{"type":"post","stable_id":"post:evals-move-from-leaderboards-to-operating-diagnostics","slug":"evals-move-from-leaderboards-to-operating-diagnostics","title":"Evals Move From Leaderboards to Operating Diagnostics","description":"Practical evaluation increasingly measures a model together with its harness, retained state, deterministic tests, review gates, and shipped artifacts.","retrieval_nugget":"A headline score without runner policy and task evidence explains less. The stronger unit is a reproducible diagnostic that can trace a failure or accepted result through the runtime.","published_at":"2026-08-14","updated_at":"2026-08-15","record_date":"2026-08-14","date_kind":"discovered_at","topics":["evals","benchmarks","harness-engineering"],"entities":["ARC-AGI-3","PR-AF","SWE-Bench ProMax","Factory"],"source_urls":["https://github.com/Agent-Field/pr-af","https://arxiv.org/abs/2608.09802","https://github.com/jerber/arc-code","https://copilotkit.ai/blog/aimock-deterministic-ai-testing","https://factory.ai/news/agent-effectiveness"],"source_format":"github","editorial_timing":{"lane":"regular_hourly","scheduled_at":"2026-08-22T11:00:00+03:00","real_news_delta":"owner-selected cross-source synthesis"},"origin":{"basket_id":"5854f7b2-5954-4d2a-8997-81596a49da74","basket_revision":1,"target_kind":"synthesis","target_id":"acd6b1d8-ad03-49e3-9f99-ecc143a0a026","owner_selection":"48","route":"hermes"},"visual_decision":{"outcome":"generate_explanatory_diagram","status":"included","reason_code":"mechanism_or_flow","explanatory_value":"operating diagnostics connect task, model, harness state, deterministic checks, review, and accepted artifact into a replayable evidence chain","text_only_limitation":"A leaderboard row cannot show which runtime component caused a result; a diagnostic pipeline makes the evidence and feedback path explicit.","owner_reviewed":true,"reviewed_by":"owner-and-codex"},"schema_version":"newruntime-agent-readable-v0.2","status":"published","visuals":[{"role":"hero","src":"/images/drip/evals-move-from-leaderboards-to-operating-diagnostics/evals-move-from-leaderboards-to-operating-diagnostics.webp","alt":"New Runtime whiteboard diagram explaining evals move from leaderboards to operating diagnostics.","caption":"New Runtime synthesis from github.com."}],"routes":{"html":"https://newruntime.com/posts/evals-move-from-leaderboards-to-operating-diagnostics/","markdown":"https://newruntime.com/posts/evals-move-from-leaderboards-to-operating-diagnostics.md","json":"https://newruntime.com/posts/evals-move-from-leaderboards-to-operating-diagnostics.json"}}