{ "schema": "certiv_public_ai_discovery_corpus_v1", "corpus_id": "certiv-ai-discovery-2026-07-16", "reviewed_at": "2026-07-16", "title": "AI-discovery evidence stress test", "selection_statement": "A curated, non-exhaustive set of public arXiv papers selected for recency, public visibility, claim diversity, and red-team value. It is not a citation ranking, endorsement, or list of proven discoveries.", "evidence_boundary": "These are Certiv-authored test dossiers, not author-issued Certiv receipts. A pinned PDF proves file identity only. A passing selected replay records one bounded observation; it does not independently reproduce a paper's central claim or establish truth, novelty, priority, safety, or correct interpretation.", "stats": { "papers": 11, "exact_pdf_versions_pinned": 11, "parser_screened_pdfs": 11, "certiv_t0_tamper_cases": 11, "selected_replay_cases": 3, "central_claims_independently_reproduced": 0 }, "selection_criteria": [ "Public arXiv record and exact-version PDF available over HTTPS.", "AI is central to the reported discovery, discovery workflow, or verification problem.", "The set spans mathematics, algorithms, materials, chemistry, biomedicine, physics, and automated AI research.", "Methodological and adversarial controls are included to test whether the evidence model resists overclaiming.", "Selection does not imply scientific endorsement, popularity rank, or successful reproduction." ], "replay_environment": { "operating_system": "Ubuntu 22.04 under WSL2", "cpu": "13th Gen Intel Core i9-13900KF", "compiler": "Ubuntu Clang 14.0.0", "isolation": "Bubblewrap with a read-only root and network namespace disabled", "review_date": "2026-07-16", "limitations": "One local environment cannot establish cross-platform portability or generalized performance. Hardware timing is reported only for the Sort3 microbenchmark." }, "core_exercise": { "status": "passed", "cases": 11, "declared_author_commands_executed": 0, "unchanged_result": "unverifiable · packed_as_unverifiable", "tampered_result": "failed · input_hash_drift", "scope": "Each exact PDF was packed as a Certiv T0 integrity input, checked unchanged, then checked after a one-byte append. This exercises identity and tamper semantics only; it does not evaluate a paper's scientific claims." }, "cases": [ { "slug": "alphaevolve", "case_type": "reported-discovery", "domain": "Algorithms and mathematics", "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery", "authors": { "display": "Alexander Novikov, Ngân Vũ, Marvin Eisenberger, et al.", "count": 18 }, "paper": { "arxiv_id": "2506.13131", "version": "v1", "submitted": "2025-06-16", "version_date": "2025-06-16", "abs_url": "https://arxiv.org/abs/2506.13131v1", "pdf_url": "https://arxiv.org/pdf/2506.13131v1", "pdf_sha256": "f092c8cbd65da89951ee6496374da9a54963f9035758c25ac6e4625f506df9ad", "pdf_bytes": 3525764, "pages": 44, "extracted_text_bytes": 137186, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A broad, high-consequence claim set with public result verifiers, including matrix multiplication and mathematical constructions.", "claim": { "reported": "The authors report that AlphaEvolve improved algorithms and mathematical constructions, including a 48-scalar-multiplication construction for 4 × 4 complex matrix multiplication.", "certiv_observed": "The exact PDF was pinned. Three selected cells from the public results notebook ran in a no-network, read-only sandbox and accepted the 48-rank construction, an Erdős step-function bound, and a dimension-11 kissing construction.", "not_established": "The AlphaEvolve agent, its search process, Google infrastructure results, the full paper, and the novelty or priority of any result were not independently reproduced." }, "verification_level": "selected-check-replayed", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v1 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity, not correctness." }, { "id": "matrix-rank-48", "status": "passed", "label": "Selected matrix verifier", "observation": "Reviewed notebook cells 4, 36, and 37 accepted a rank-48 decomposition for the 4 × 4 × 4 matrix multiplication tensor.", "boundary": "This checks the supplied construction; it does not reproduce how AlphaEvolve found it or establish novelty." }, { "id": "math-constructions", "status": "passed", "label": "Two selected mathematical verifiers", "observation": "Reviewed cells 77–79 produced C5 ≤ 0.38092303510845016, and cells 113–114 accepted a dimension-11 kissing construction of at least 593.", "boundary": "These are selected executable checks from the authors' result notebook, not a review of every result." }, { "id": "full-notebook", "status": "blocked", "label": "Full notebook portability", "observation": "A sequential replay stopped at cell 72 because Python 3.9 lacks sys.set_int_max_str_digits.", "boundary": "Environment drift made this run inconclusive; it is not evidence that the mathematical result failed." } ], "artifacts": [ { "label": "AlphaEvolve results repository", "url": "https://github.com/google-deepmind/alphaevolve_results", "observed_revision": "4226acbf237ff9ad10ba7673a2af127a2d8a5971", "sha256": "2cce2543e48c89aa3e91614272a698a0147dd2548ea11cf92f1292b7435d38ff", "note": "SHA-256 is for mathematical_results.ipynb at the observed revision. The Certiv cell-selection runner SHA-256 is 82a2424c064ff5a6e2a75d8291c8d9b65577b9d2243580e4e467934fe5f9f73c. The repository publishes results and verifiers, not the AlphaEvolve engine." } ], "red_team_findings": [ "The public repository does not expose the discovery engine, so generation claims cannot be replayed from this artifact.", "A verifier accepting an author-supplied construction is stronger than prose but narrower than reproducing the search or establishing priority.", "The notebook has a Python-version portability dependency that was not declared as a locked environment." ], "certiv_mapping": { "inputs": "Exact PDF plus a commit-pinned public notebook.", "checks": "Three reviewed, isolated notebook slices with exact observed output.", "receipt_boundary": "selected-check-replayed; central claim not independently reproduced", "missing_for_deeper_verification": "The discovery engine, locked full environment, infrastructure workloads, and an independent novelty review." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2506.13131v1" }, { "label": "Public results repository", "url": "https://github.com/google-deepmind/alphaevolve_results" } ] }, { "slug": "mathematical-exploration-at-scale", "case_type": "reported-discovery", "domain": "Mathematics", "title": "Mathematical exploration and discovery at scale", "authors": { "display": "Bogdan Georgiev, Javier Gómez-Serrano, Terence Tao, and Adam Zsolt Wagner", "count": 4 }, "paper": { "arxiv_id": "2511.02864", "version": "v3", "submitted": "2025-11-03", "version_date": "2025-12-22", "abs_url": "https://arxiv.org/abs/2511.02864v3", "pdf_url": "https://arxiv.org/pdf/2511.02864v3", "pdf_sha256": "77b43844077c98c26dfc76ad391cadbbd53d48a1d290ddae575ff4d1bcdef9d2", "pdf_bytes": 12141713, "pages": 81, "extracted_text_bytes": 269596, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A recent 67-problem mathematical study that exposes concrete constructions and makes the gap between checking examples and reproducing a whole study especially visible.", "claim": { "reported": "The authors report that AlphaEvolve rediscovered many best-known solutions and improved several mathematical constructions across 67 problems.", "certiv_observed": "The exact v3 paper was pinned. Selected public verifiers for an Erdős step-function bound and a dimension-11 kissing construction ran successfully in isolation.", "not_established": "The 67-problem campaign, system autonomy, comparative preparation time, novelty, generality, and all unselected constructions were not independently reproduced." }, "verification_level": "selected-check-replayed", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v3 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity, not mathematical validity." }, { "id": "erdos-step-function", "status": "passed", "label": "Selected Erdős verifier", "observation": "Reviewed notebook cells 77–79 accepted the supplied step function and produced C5 ≤ 0.38092303510845016.", "boundary": "One construction was checked; the surrounding theorem, comparative literature, and novelty were not adjudicated." }, { "id": "kissing-number-11", "status": "passed", "label": "Selected kissing-number verifier", "observation": "Reviewed notebook cells 113–114 accepted the supplied construction showing at least 593 spheres in dimension 11.", "boundary": "This validates the supplied witness under the notebook's check, not the paper's full campaign." }, { "id": "all-67-problems", "status": "not-run", "label": "Full study reproduction", "observation": "No end-to-end replay of all 67 problems was attempted.", "boundary": "The dossier must not be read as 67 independent reproductions." } ], "artifacts": [ { "label": "AlphaEvolve mathematical results repository", "url": "https://github.com/google-deepmind/alphaevolve_results", "observed_revision": "4226acbf237ff9ad10ba7673a2af127a2d8a5971", "sha256": "2cce2543e48c89aa3e91614272a698a0147dd2548ea11cf92f1292b7435d38ff", "note": "SHA-256 is for mathematical_results.ipynb at the observed revision. The Certiv cell-selection runner SHA-256 is 82a2424c064ff5a6e2a75d8291c8d9b65577b9d2243580e4e467934fe5f9f73c." } ], "red_team_findings": [ "Two accepted witnesses cannot support a claim that the entire 67-problem study was reproduced.", "The results repository supplies checkable outputs but not the proprietary discovery engine.", "Literature comparison and novelty require domain-expert review outside a computational receipt." ], "certiv_mapping": { "inputs": "Exact paper version, repository revision, and notebook hash.", "checks": "Two selected witness verifiers with recorded cells and outputs.", "receipt_boundary": "selected-check-replayed; full mathematical study not reproduced", "missing_for_deeper_verification": "All problem environments, generation system, complete replay plan, and independent mathematical and novelty review." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2511.02864v3" }, { "label": "Public results repository", "url": "https://github.com/google-deepmind/alphaevolve_results" } ] }, { "slug": "co-scientist", "case_type": "reported-discovery", "domain": "Biomedicine", "title": "Accelerating scientific discovery with Co-Scientist", "authors": { "display": "Juraj Gottweis, Wei-Hung Weng, Alexander Daryin, et al.", "count": 51 }, "paper": { "arxiv_id": "2502.18864", "version": "v2", "submitted": "2025-02-26", "version_date": "2026-06-29", "abs_url": "https://arxiv.org/abs/2502.18864v2", "pdf_url": "https://arxiv.org/pdf/2502.18864v2", "pdf_sha256": "e603a77c9d2bdfb528313e4cf35405b72261e018a70da724fcebca94c96e4716", "pdf_bytes": 5885207, "pages": 157, "extracted_text_bytes": 348722, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A recent biomedical case with experimental claims and a deliberately difficult reproducibility boundary around private data and infrastructure.", "claim": { "reported": "The authors report that a Gemini-based multi-agent system generated biomedical hypotheses, including drug-repurposing and combination-therapy candidates supported by experimental validation.", "certiv_observed": "The exact v2 paper was pinned and its disclosure boundary reviewed. The paper states that three real-world validation datasets and the full system source are not public.", "not_established": "The biomedical experiments, hypotheses, model behavior, private datasets, and proprietary system were not independently reproduced or clinically validated by Certiv." }, "verification_level": "source-pinned", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v2 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "disclosure-review", "status": "observed", "label": "Availability boundary", "observation": "The exact paper says the real-world validation data are not public and the full source depends on proprietary internal infrastructure and resources.", "boundary": "Pseudocode and prompts help inspection but cannot substitute for the withheld system and data." }, { "id": "biomedical-replay", "status": "blocked", "label": "Experimental reproduction", "observation": "No safe, authorized path to the private datasets, lab protocols, samples, or full system was available.", "boundary": "Experimental results remain author-reported in this dossier." } ], "artifacts": [], "red_team_findings": [ "The strongest claims depend on wet-lab evidence and data that are not available for public replay.", "Pseudocode cannot demonstrate equivalence to a proprietary deployed multi-agent system.", "Biomedical hypothesis generation is not evidence of clinical utility, safety, or treatment efficacy." ], "certiv_mapping": { "inputs": "Exact paper version and its stated availability disclosures.", "checks": "Identity and disclosure review only; no experimental command was available.", "receipt_boundary": "source-pinned; biomedical results remain author-reported", "missing_for_deeper_verification": "Public datasets, complete system source, locked model and prompt identities, lab protocol, raw measurements, and independent experimental replication." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2502.18864v2" }, { "label": "Nature article", "url": "https://doi.org/10.1038/s41586-026-10644-y" } ] }, { "slug": "mattergen", "case_type": "reported-discovery", "domain": "Materials science", "title": "MatterGen: a generative model for inorganic materials design", "authors": { "display": "Claudio Zeni, Robert Pinsler, Daniel Zügner, et al.", "count": 21 }, "paper": { "arxiv_id": "2312.03687", "version": "v2", "submitted": "2023-12-06", "version_date": "2024-01-29", "abs_url": "https://arxiv.org/abs/2312.03687v2", "pdf_url": "https://arxiv.org/pdf/2312.03687v2", "pdf_sha256": "891e691d87c7c3f63dffa043beb869ad5ed14759dcf3a07df7975906d5ec2030", "pdf_bytes": 12150197, "pages": 56, "extracted_text_bytes": 139476, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A prominent generative-materials case where public code exists but later checkpoints and licensed structural data do not exactly match the paper experiment.", "claim": { "reported": "The authors report that MatterGen generates stable, novel inorganic structures and can be steered toward multiple property constraints.", "certiv_observed": "The exact v2 PDF was pinned and the official code repository was commit-inventoried. Its documentation warns that released checkpoints were retrained and may not reproduce paper values; some structures also cannot be shared for licensing reasons.", "not_established": "Material stability, novelty, synthesis feasibility, property accuracy, and the paper's comparative performance were not independently reproduced." }, "verification_level": "artifact-inventoried", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v2 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "repository-inventory", "status": "observed", "label": "Official repository inventory", "observation": "The public repository was pinned at commit ac9ddd406171138c3f037d06b9b53fedbbb1c536.", "boundary": "A repository revision is not a successful reproduction." }, { "id": "paper-equivalent-replay", "status": "blocked", "label": "Paper-equivalent replay", "observation": "The released checkpoints are documented as retrained rather than identical, and some ICSD structures are unavailable under licensing limits.", "boundary": "A run of the released package would not automatically be a reproduction of the paper experiment." } ], "artifacts": [ { "label": "Official MatterGen repository", "url": "https://github.com/microsoft/mattergen", "observed_revision": "ac9ddd406171138c3f037d06b9b53fedbbb1c536", "note": "Repository documentation says released checkpoints were retrained and can differ from paper results." } ], "red_team_findings": [ "Released checkpoints are not the exact paper checkpoints, which prevents bit-for-bit experiment equivalence.", "Licensing limits remove part of the structural evaluation data from the public package.", "The repository's faster ML-force-field evaluation is not equivalent to the paper's DFT evaluation.", "Computational stability predictions do not establish successful physical synthesis." ], "certiv_mapping": { "inputs": "Exact PDF and commit-pinned official repository.", "checks": "Artifact and disclosure inventory; no expensive model or DFT run.", "receipt_boundary": "artifact-inventoried; paper-equivalent replay unavailable", "missing_for_deeper_verification": "Exact checkpoints, complete licensed data, locked dependencies, DFT workflow, compute budget, and independent synthesis or property validation." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2312.03687v2" }, { "label": "Official repository", "url": "https://github.com/microsoft/mattergen" }, { "label": "Nature article and availability statements", "url": "https://doi.org/10.1038/s41586-025-08628-5" } ] }, { "slug": "ai-scientist-v1", "case_type": "discovery-system", "domain": "Automated machine-learning research", "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery", "authors": { "display": "Chris Lu, Cong Lu, Robert Tjarko Lange, Jakob Foerster, Jeff Clune, and David Ha", "count": 6 }, "paper": { "arxiv_id": "2408.06292", "version": "v3", "submitted": "2024-08-12", "version_date": "2024-09-01", "abs_url": "https://arxiv.org/abs/2408.06292v3", "pdf_url": "https://arxiv.org/pdf/2408.06292v3", "pdf_sha256": "e911f90b0114d5e3fc23a0000177ea670fa9ed79d5f9a664efa2980fdb350507", "pdf_bytes": 11731143, "pages": 186, "extracted_text_bytes": 510675, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "An end-to-end automated-research system whose evaluation language tests whether Certiv distinguishes a simulated review threshold from real peer review or scientific correctness.", "claim": { "reported": "The authors report an automated workflow that generates ideas, edits code, runs experiments, writes papers, and evaluates them with an automated reviewer.", "certiv_observed": "The exact v3 PDF and public repository revision were pinned. The paper's headline acceptance language is explicitly based on an automated reviewer's threshold.", "not_established": "The generated papers were not shown by this dossier to be correct, novel, reproducible, or accepted by a real conference review process. The full workflow was not replayed." }, "verification_level": "artifact-inventoried", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v3 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "repository-inventory", "status": "observed", "label": "Public code inventory", "observation": "The official repository was pinned at commit 1de1dbc1f4ee2c5f61e9c94348d55eb51d7fa2eb.", "boundary": "The repository still requires models, API credentials, GPU resources, and external services." }, { "id": "end-to-end-replay", "status": "not-run", "label": "End-to-end autonomous research run", "observation": "No model-funded, GPU-backed workflow was executed.", "boundary": "No generated paper or cost claim was reproduced." } ], "artifacts": [ { "label": "Official AI Scientist repository", "url": "https://github.com/SakanaAI/AI-Scientist", "observed_revision": "1de1dbc1f4ee2c5f61e9c94348d55eb51d7fa2eb", "note": "Open code still depends on third-party models, APIs, external services, and substantial compute." } ], "red_team_findings": [ "Passing an automated review threshold is not the same event as acceptance by human peer review.", "A generated manuscript is an output artifact, not proof that its scientific statements are true.", "Cost and autonomy claims depend on model versions, service behavior, templates, and external resources that can drift." ], "certiv_mapping": { "inputs": "Exact paper and commit-pinned public code.", "checks": "Identity, repository, dependency, and claim-semantics review.", "receipt_boundary": "artifact-inventoried; no end-to-end run", "missing_for_deeper_verification": "Pinned model identities, prompts, API behavior, templates, datasets, GPU environment, cost ledger, and independent review of each generated paper." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2408.06292v3" }, { "label": "Official repository", "url": "https://github.com/SakanaAI/AI-Scientist" } ] }, { "slug": "ai-scientist-v2", "case_type": "discovery-system", "domain": "Automated machine-learning research", "title": "The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search", "authors": { "display": "Yutaro Yamada, Robert Tjarko Lange, Cong Lu, et al.", "count": 8 }, "paper": { "arxiv_id": "2504.08066", "version": "v1", "submitted": "2025-04-10", "version_date": "2025-04-10", "abs_url": "https://arxiv.org/abs/2504.08066v1", "pdf_url": "https://arxiv.org/pdf/2504.08066v1", "pdf_sha256": "53bafd3028e3f8829a3d85220e84dcf0d18934f9b75c092a60de303ff3644bd2", "pdf_bytes": 8923691, "pages": 69, "extracted_text_bytes": 195670, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A later automated-research system with a real workshop submission outcome, useful for separating process facts from scientific validation.", "claim": { "reported": "The authors report a more autonomous research agent and say one of three fully AI-generated workshop submissions was accepted after peer review.", "certiv_observed": "The exact v1 PDF and official repository revision were pinned. The paper documents a workshop acceptance event, which is a process outcome rather than a reproduction of the paper's scientific conclusions.", "not_established": "Acceptance does not establish that the generated research is true, novel, reproducible, or representative of general automated-science performance. The system was not replayed here." }, "verification_level": "artifact-inventoried", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v1 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "repository-inventory", "status": "observed", "label": "Public code inventory", "observation": "The official v2 repository was pinned at commit 96bd51617cfdbb494a9fc283af00fe090edfae48.", "boundary": "Open code is not a completed run and still relies on mutable models and services." }, { "id": "end-to-end-replay", "status": "not-run", "label": "End-to-end tree-search run", "observation": "No GPU- and API-backed autonomous research campaign was executed.", "boundary": "Autonomy, cost, output quality, and acceptance rate were not reproduced." } ], "artifacts": [ { "label": "Official AI Scientist-v2 repository", "url": "https://github.com/SakanaAI/AI-Scientist-v2", "observed_revision": "96bd51617cfdbb494a9fc283af00fe090edfae48", "note": "The workflow depends on external model APIs, GPU workloads, and changing service behavior." } ], "red_team_findings": [ "One workshop acceptance cannot establish a general success rate for autonomous scientific discovery.", "Peer-review acceptance is meaningful process evidence but is not itself a truth or reproducibility certificate.", "Externally hosted models and services weaken long-term replay unless their behavior and outputs are separately captured." ], "certiv_mapping": { "inputs": "Exact paper and commit-pinned public repository.", "checks": "Identity, artifact, dependency, and claim-boundary review.", "receipt_boundary": "artifact-inventoried; workshop outcome not treated as scientific proof", "missing_for_deeper_verification": "Pinned models and services, full prompts, datasets, compute environment, generated-paper artifacts, reviewer record, and independent reproduction of scientific results." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2504.08066v1" }, { "label": "Official repository", "url": "https://github.com/SakanaAI/AI-Scientist-v2" } ] }, { "slug": "ai-newton", "case_type": "discovery-system", "domain": "Physics", "title": "AI-Newton: A Concept-Driven Physical Law Discovery System without Prior Physical Knowledge", "authors": { "display": "You-Le Fang, Dong-Shan Jian, Xiang Li, and Yan-Qing Ma", "count": 4 }, "paper": { "arxiv_id": "2504.01538", "version": "v2", "submitted": "2025-04-02", "version_date": "2025-12-11", "abs_url": "https://arxiv.org/abs/2504.01538v2", "pdf_url": "https://arxiv.org/pdf/2504.01538v2", "pdf_sha256": "bce6758e3ec33fe715b2df7132733107f98f1e928abb9ead89b9b7e2b3a8cdc1", "pdf_bytes": 5198153, "pages": 6, "extracted_text_bytes": 28334, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A public concept-driven physics system whose proprietary symbolic dependency and multi-day benchmark make resource and licensing boundaries concrete.", "claim": { "reported": "The authors report that AI-Newton derived general physical laws from raw multi-experiment data without prior physical knowledge, rediscovering several established laws.", "certiv_observed": "The exact v2 PDF and repository revision were pinned. The documented stack requires Python, Rust, CUDA, and Maple 2024; Maple was not present in the review environment.", "not_established": "The benchmark, days-long full run, absence of prior knowledge, robustness to noise, and general discovery capability were not independently reproduced. The reported outputs are rediscoveries of known laws, not newly established laws of nature." }, "verification_level": "artifact-inventoried", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v2 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "repository-inventory", "status": "observed", "label": "Repository and dependency inventory", "observation": "The repository was pinned at c143e865be42e067faf64cd8117cbeffd0fccfb6; its README, Cargo.lock, and sample knowledge file were separately hashed.", "boundary": "Inventory records what was reviewed, not whether the system works as claimed." }, { "id": "full-benchmark", "status": "blocked", "label": "Full benchmark replay", "observation": "Maple 2024 was unavailable, and the documented full test requires several days on high-end servers.", "boundary": "The missing proprietary dependency and compute budget prevented a faithful run." } ], "artifacts": [ { "label": "AI-Newton repository", "url": "https://github.com/Science-Discovery/AI-Newton", "observed_revision": "c143e865be42e067faf64cd8117cbeffd0fccfb6", "note": "README SHA-256 e35a6fd49f3dfa9600b400d906ef46146e3ec9d0c179c985a313b89fbfe2a4ab; Cargo.lock SHA-256 1e75c6930405e1ec20be9936c6ca9d75e8b42f3d8a61462d33a18d244cf8920c; sample knowledge SHA-256 a7524f42a48dc6165eabd65c9fc3696fdd652df8dc1e4d251bdc93ebdb956df9." } ], "red_team_findings": [ "A proprietary Maple dependency prevents a fully open replay even though the repository is public.", "The short benchmark and full multi-day benchmark have materially different resource requirements.", "Rediscovering known laws is evidence about a system's behavior, not a new physical discovery." ], "certiv_mapping": { "inputs": "Exact PDF, commit-pinned code, and hashes for key dependency and knowledge files.", "checks": "Artifact and prerequisite inventory; no benchmark execution.", "receipt_boundary": "artifact-inventoried; replay blocked by Maple and compute requirements", "missing_for_deeper_verification": "Licensed Maple environment, exact CUDA stack, experiment data, complete locked environment, multi-day compute, and independent evaluation of recovered laws." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2504.01538v2" }, { "label": "Public repository", "url": "https://github.com/Science-Discovery/AI-Newton" } ] }, { "slug": "coscientist-chemistry", "case_type": "discovery-system", "domain": "Chemistry and laboratory automation", "title": "Emergent autonomous scientific research capabilities of large language models", "authors": { "display": "Daniil A. Boiko, Robert MacKnight, and Gabe Gomes", "count": 3 }, "paper": { "arxiv_id": "2304.05332", "version": "v1", "submitted": "2023-04-11", "version_date": "2023-04-11", "abs_url": "https://arxiv.org/abs/2304.05332v1", "pdf_url": "https://arxiv.org/pdf/2304.05332v1", "pdf_sha256": "7adaf953cbb194b787f7861dcc6bce2c943e86bea34ab5b011368325fc59fb30", "pdf_bytes": 2868061, "pages": 48, "extracted_text_bytes": 95210, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "An influential early laboratory-agent case that stress-tests safety, physical execution, and deferred-code-release boundaries.", "claim": { "reported": "The authors report an LLM-based agent that planned and executed scientific tasks, including catalyzed cross-coupling reactions using laboratory automation.", "certiv_observed": "The exact v1 paper was pinned. This version says code, data, and prompts would be released later and discusses safety reasons for limiting disclosure.", "not_established": "The laboratory execution, agent autonomy, reaction outcomes, safety controls, and model behavior were not independently reproduced. The demonstrated reaction was known chemistry rather than a new chemical discovery." }, "verification_level": "source-pinned", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v1 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "availability-review", "status": "observed", "label": "Exact-version availability", "observation": "Version 1 defers release of code, data, and prompts and describes a safety rationale.", "boundary": "Later artifacts, if any, cannot silently be treated as artifacts bundled with v1." }, { "id": "lab-replay", "status": "blocked", "label": "Laboratory replay", "observation": "No complete v1 system package, authorized robot lab, chemicals, or experiment protocol was available.", "boundary": "The physical execution remains author-reported." } ], "artifacts": [], "red_team_findings": [ "The exact paper version did not ship the code, data, and prompts required for a computational replay.", "Executing chemistry entails physical and dual-use risks that a software sandbox does not contain.", "Completing a known reaction demonstrates task execution, not necessarily scientific discovery." ], "certiv_mapping": { "inputs": "Exact v1 paper and its disclosure statements.", "checks": "Identity and availability review only.", "receipt_boundary": "source-pinned; laboratory claims not replayed", "missing_for_deeper_verification": "Exact prompts, models, code, robot configuration, chemicals, raw sensor data, safety approvals, and independent laboratory replication." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2304.05332v1" } ] }, { "slug": "ai-assisted-theoretical-physics", "case_type": "reported-discovery", "domain": "Theoretical physics and mathematics", "title": "Solving an Open Problem in Theoretical Physics using AI-Assisted Discovery", "authors": { "display": "Michael P. Brenner, Vincent Cohen-Addad, and David P. Woodruff", "count": 3 }, "paper": { "arxiv_id": "2603.04735", "version": "v1", "submitted": "2026-03-05", "version_date": "2026-03-05", "abs_url": "https://arxiv.org/abs/2603.04735v1", "pdf_url": "https://arxiv.org/pdf/2603.04735v1", "pdf_sha256": "70fce2b699eb79312cffa7b39d3cad07abb5e116d25dec5b901cd0768ff68858", "pdf_bytes": 699380, "pages": 22, "extracted_text_bytes": 47414, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A very recent open-problem claim with analytical derivations, useful for separating inspectable exposition from an executable proof or independent expert validation.", "claim": { "reported": "The authors report that an AI-assisted neuro-symbolic workflow derived exact analytical solutions for a cosmic-string gravitational-radiation integral that they characterize as an open problem.", "certiv_observed": "The exact v1 paper was pinned and its prompts, search description, and derivations are inspectable in the paper. No public runnable artifact linked from the exact record was identified in this review.", "not_established": "The open-problem status, derivation correctness, novelty, system autonomy, and agreement with all relevant numerical or physical constraints were not independently adjudicated." }, "verification_level": "source-pinned", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v1 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "artifact-search", "status": "observed", "label": "Linked-artifact review", "observation": "No public runnable code or proof-checker artifact was identified from the exact arXiv record and paper during this review.", "boundary": "Absence from this review is not proof that no artifact exists elsewhere." }, { "id": "derivation-review", "status": "not-run", "label": "Independent derivation and domain review", "observation": "No formal proof checker, independent symbolic derivation, or specialist peer assessment was completed.", "boundary": "The central result remains author-reported in this dossier." } ], "artifacts": [], "red_team_findings": [ "A detailed derivation is inspectable evidence but not automatically a machine-checkable proof.", "Calling a problem open and a solution novel requires an independent literature and domain review.", "Prompts and workflow descriptions do not by themselves reproduce model behavior." ], "certiv_mapping": { "inputs": "Exact paper version and the derivations and prompts contained within it.", "checks": "Identity and linked-artifact review only.", "receipt_boundary": "source-pinned; analytical result not independently checked", "missing_for_deeper_verification": "Executable symbolic checks, raw search traces, pinned model behavior, numerical comparison data, and independent theoretical-physics review." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2603.04735v1" } ] }, { "slug": "verification-in-ai-discovery", "case_type": "method-control", "domain": "Scientific method", "title": "The Need for Verification in AI-Driven Scientific Discovery", "authors": { "display": "Cristina Cornelio, Takuya Ito, Ryan Cory-Wright, Sanjeeb Dash, and Lior Horesh", "count": 5 }, "paper": { "arxiv_id": "2509.01398", "version": "v2", "submitted": "2025-09-01", "version_date": "2025-12-17", "abs_url": "https://arxiv.org/abs/2509.01398v2", "pdf_url": "https://arxiv.org/pdf/2509.01398v2", "pdf_sha256": "d598af4bcb9b0546314e1718b1e94e506709a225602f315cbb59f62317dd9ce4", "pdf_bytes": 2009058, "pages": 25, "extracted_text_bytes": 86104, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A methodological control included to test whether the corpus itself states a defensible verification boundary rather than optimizing for positive AI-discovery narratives.", "claim": { "reported": "The authors argue that scalable, reliable verification must be central to AI-assisted scientific discovery.", "certiv_observed": "The exact v2 paper was pinned and treated as a methodological control, not as an experimental discovery result.", "not_established": "Pinning this review does not prove its historical, economic, or methodological arguments, nor does it validate Certiv's implementation." }, "verification_level": "source-pinned", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v2 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "method-control", "status": "control", "label": "Methodological control", "observation": "The case is intentionally classified as analysis of verification needs rather than an AI-made discovery.", "boundary": "Its inclusion prevents the corpus label from implying that every paper reports a new discovery." } ], "artifacts": [], "red_team_findings": [ "A verification framework must expose unverifiable and blocked cases, not only passing demonstrations.", "Computational verification is only one layer; empirical, formal, novelty, and interpretation review can remain outside its scope.", "The word verification is overloaded, so every dossier needs an operational statement of what was actually checked." ], "certiv_mapping": { "inputs": "Exact review-paper version.", "checks": "Identity and classification control.", "receipt_boundary": "methodological control; no discovery claim", "missing_for_deeper_verification": "Independent review of the paper's synthesis and any empirical claims it cites." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2509.01398v2" } ] }, { "slug": "sort3-response", "case_type": "adversarial-response", "domain": "Algorithms and systems performance", "title": "Shorter and faster than Sort3AlphaDev", "authors": { "display": "Cassio Neri", "count": 1 }, "paper": { "arxiv_id": "2307.14503", "version": "v1", "submitted": "2023-07-23", "version_date": "2023-07-23", "abs_url": "https://arxiv.org/abs/2307.14503v1", "pdf_url": "https://arxiv.org/pdf/2307.14503v1", "pdf_sha256": "7359b5844b5826bb269b3e552bd8f57b53c1c2854ba4aaea9c525fd4b6929b34", "pdf_bytes": 67617, "pages": 5, "extracted_text_bytes": 8185, "parser_observations": [ "Recognized as a PDF and not encrypted.", "pdfinfo reported JavaScript: no.", "pdfdetach reported no embedded files.", "pdftotext completed successfully." ] }, "selection_rationale": "A direct adversarial response to an AI-discovered algorithm, selected to test negative evidence, portability, test adequacy, and performance-claim scope.", "claim": { "reported": "The author presents shorter Sort3 implementations and a C/C++ implementation claimed to be faster than the published AlphaDev version.", "certiv_observed": "Eight supplied sorting functions passed 17,173,580 cases in an independent harness. On one i9-13900KF environment with Clang 14, a nine-trial microbenchmark measured Sort3_faster at a 0.837 ns/call median versus 1.031 ns/call for Sort3AlphaDev.", "not_established": "The run does not prove correctness over all machine states, universal instruction minimality, or generalized speed across CPUs, compilers, workloads, and measurement methods." }, "verification_level": "selected-check-replayed", "central_claim_independently_reproduced": false, "checks": [ { "id": "pdf-identity", "status": "observed", "label": "Exact PDF identity", "observation": "The downloaded v1 PDF matched the recorded SHA-256, byte count, and page count.", "boundary": "This establishes file identity only." }, { "id": "author-test", "status": "passed", "label": "Author test under Clang", "observation": "The repository's supplied test program compiled with Clang and returned success.", "boundary": "Its 'exhaustive' label covers only 13 explicit cases, and its process exit status does not fail when an individual function reports failure." }, { "id": "independent-functional-harness", "status": "passed", "label": "Independent functional stress test", "observation": "Eight functions passed 17,173,580 deterministic cases: signed domains for five functions, nonnegative domains for three documented variants, plus edge cases.", "boundary": "This is broad finite-domain evidence, not a proof for every possible integer triple or target architecture." }, { "id": "gcc-portability", "status": "failed", "label": "GCC 11.4 portability probe", "observation": "Direct compilation failed on the inline-assembly movsb destination syntax; Clang 14 compiled it.", "boundary": "This is a portability finding, not a functional counterexample under the documented Clang-compatible path." }, { "id": "local-microbenchmark", "status": "passed", "label": "Bounded one-host timing comparison", "observation": "Across nine 5,000,000-call trials: Sort3_faster median 0.837 ns/call (0.794–0.863); Sort3AlphaDev median 1.031 (0.983–1.131). Checksums matched.", "boundary": "One virtualized host, compiler, optimization mode, and deterministic input stream cannot establish universal performance." } ], "artifacts": [ { "label": "Sort3 response repository", "url": "https://github.com/cassioneri/sort3", "observed_revision": "431ac0efcc216792f10e8f23d24a8ee387b3c348", "sha256": "b99439b6013fadde20b57fafe40643913a0bb0a2c48d93659865c7f81344fdee", "note": "SHA-256 is for sort.h. The independent harness SHA-256 is 4475c66c5030ccf3949d2bdb0aa51e2ab8115d94487d77049bfc66788a02d551; benchmark harness SHA-256 is 4b5464cc80a8f3684813b36403db97d9b9b6b3925fa56262a465eb85fa08edd3." } ], "red_team_findings": [ "The repository's test is not exhaustive despite its label and has weak failure propagation.", "Sort3_15 is documented in the repository erratum as valid only for nonnegative inputs.", "The inline assembly compiled under Clang 14 but failed under GCC 11.4, exposing a portability boundary.", "Performance evidence must state CPU, compiler, input distribution, trial count, and measurement scope." ], "certiv_mapping": { "inputs": "Exact paper, commit-pinned source, source hash, and two independently hashed harnesses.", "checks": "Author test, 17,173,580-case functional harness, compiler portability probe, and nine-trial local microbenchmark.", "receipt_boundary": "selected-check-replayed; finite functional and one-host performance evidence only", "missing_for_deeper_verification": "Formal correctness proof, exact instruction-count audit, multi-compiler and multi-architecture matrix, controlled bare-metal benchmarking, and independent minimality analysis." }, "sources": [ { "label": "Exact arXiv record", "url": "https://arxiv.org/abs/2307.14503v1" }, { "label": "Response repository", "url": "https://github.com/cassioneri/sort3" } ] } ] }