{"type":"analysis","id":"analysis:research-agent-reproductions-of-icml-2026-scaled-audits-not-autonomous-review","slug":"research-agent-reproductions-of-icml-2026-scaled-audits-not-autonomous-review","title":"Agent reproductions of ICML 2026 scaled audits, not autonomous review","description":"Hugging Face and alphaXiv ran a 19-day challenge in which coding agents tried 2,226 ICML 2026 papers claim by claim. About half of examined papers had at least one verified claim; nearly a quarter had a falsified or contested claim; humans still re-checked the failures that mattered.","published_at":"2026-11-08T01:00:00.000Z","updated_at":"2026-11-08T01:00:00.000Z","record_date":"2026-11-08","topics":["agents","coding_agents","research"],"source_urls":["https://huggingface.co/blog/icml-2026-open-reproductions","https://huggingface.co/datasets/ICML-2026-agent-repro/verdicts","https://www.science.org/content/article/researchers-presented-6000-papers-major-meeting-could-ai-reproduce-their-findings","https://huggingface.co/spaces/ICML-2026-agent-repro/challenge/blob/main/PROMPT.md","https://icml-2026-agent-repro-challenge.static.hf.space/faq.html","https://huggingface.co/ICML-2026-agent-repro","https://huggingface.co/zai-org/GLM-5.2","https://huggingface.co/blog/ParetoOptimal/final-report-icml2026","https://sai.science/blog/how-much-science-is-verifiable","https://media.icml.cc/Conferences/ICML2026/ICML2026_Fact_Sheet.pdf","https://icml-2026-agent-repro-challenge.static.hf.space/index.html","https://icml-2026-agent-repro-challenge.static.hf.space/leaderboard.html","https://huggingface.co/docs/trackio","https://huggingface.co/docs/hub/jobs","https://blog.icml.cc/2026/01/08/whats-new-in-icml-2026-peer-review/"],"language":"en","routes":{"html":"https://newruntime.com/analysis/research-agent-reproductions-of-icml-2026-scaled-audits-not-autonomous-review/","markdown":"https://newruntime.com/analysis/research-agent-reproductions-of-icml-2026-scaled-audits-not-autonomous-review.md","json":"https://newruntime.com/analysis/research-agent-reproductions-of-icml-2026-scaled-audits-not-autonomous-review.json"},"author":{"role":"author","name":"Andrey Reshetnikov","profile":"https://newruntime.com/owner-profile.md","profile_json":"https://newruntime.com/owner-profile.json","same_as":["https://www.linkedin.com/in/reshetnikov1/","https://t.me/lapetuse"]},"editorial_provenance":{"desk":"authorial","target_ref":"deep_research:b3321910-5888-47a8-ae83-77223df4ef14","authorial_article_id":"b3321910-5888-47a8-ae83-77223df4ef14"},"next_reads":[{"type":"topic","path":"/topics/agents/","url":"https://newruntime.com/topics/agents/","title":"Agents","reason":"Explore the agents topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/agents.md","json_url":"https://newruntime.com/topics/agents.json"},{"type":"topic","path":"/topics/coding-agents/","url":"https://newruntime.com/topics/coding-agents/","title":"Coding Agents","reason":"Explore the coding_agents topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/coding-agents.md","json_url":"https://newruntime.com/topics/coding-agents.json"},{"type":"collection","path":"/analysis/","url":"https://newruntime.com/analysis/","title":"Analysis","reason":"Open the latest authorial analysis.","media_type":"text/html","json_url":"https://newruntime.com/analysis.json"},{"type":"collection","path":"/sources/","url":"https://newruntime.com/sources/","title":"Sources","reason":"See the original sources this archive cites.","media_type":"text/html"}]}