{"type":"post","id":"ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e","slug":"ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e","title":"Ai2 released StateBench, a benchmark splitting state tracking into four algebraic complexity tiers.","description":"Ai2 has released StateBench, an open benchmark and generation toolkit for evaluating how language models track changing state across sequences of assignments.","retrieval_nugget":"Ai2 has released StateBench, an open benchmark and generation toolkit for evaluating how language models track changing state across sequences of assignments. Its four task families vary in algebraic complexity while controlling several properties of the underlying traces, giving researchers a more structured way to test where state tracking breaks down.","published_at":"2026-11-04","updated_at":"2026-11-07","record_date":"2026-11-04","date_kind":"published_at","topics":["models"],"entities":["Ai2","StateBench","OLMo 3","OLMo Hybrid"],"related_patterns":["ai-engineering-evaluations-are-splitting-into-task-specific"],"novelty_label":"same_event","prior_refs":["https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-afffa2/","https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-f7c04b/"],"source_url":"https://github.com/allenai/state-bench","source_urls":["https://github.com/allenai/state-bench"],"schema_version":"newruntime-agent-readable-v0.2","stable_id":"post:ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e","status":"published","visuals":[],"editor":{"role":"editor","name":"Andrey Reshetnikov","profile":"https://newruntime.com/owner-profile.md","profile_json":"https://newruntime.com/owner-profile.json","same_as":["https://www.linkedin.com/in/reshetnikov1/","https://t.me/lapetuse"]},"routes":{"html":"https://newruntime.com/posts/ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e/","markdown":"https://newruntime.com/posts/ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e.md","json":"https://newruntime.com/posts/ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e.json"},"markdown_url":"https://newruntime.com/posts/ai2-released-statebench-a-benchmark-splitting-state-tracking-2c9c2e.md","next_reads":[{"type":"topic","path":"/topics/models/","url":"https://newruntime.com/topics/models/","title":"Models","reason":"Explore the models topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/models.md","json_url":"https://newruntime.com/topics/models.json"},{"type":"collection","path":"/posts/","url":"https://newruntime.com/posts/","title":"Field Notes","reason":"Open the latest editorial analysis.","media_type":"text/html","json_url":"https://newruntime.com/posts.json"},{"type":"collection","path":"/sources/","url":"https://newruntime.com/sources/","title":"Sources","reason":"See the original sources this archive cites.","media_type":"text/html"}]}