{"type":"post","id":"statebench-testing-how-language-models-track-state-changes-f7c04b","slug":"statebench-testing-how-language-models-track-state-changes-f7c04b","title":"StateBench: Testing how language models track state changes over long documents","description":"Ai2 introduced StateBench, a benchmark that tests how models track changing entity states across long sequences, from simple reassignment to dependencies that require carrying updates forward.","retrieval_nugget":"Ai2 introduced StateBench, a benchmark that tests how models track changing entity states across long sequences, from simple reassignment to dependencies that require carrying updates forward. Its results show that strong performance on easier state questions does not guarantee reliable tracking of harder updates as inputs grow, giving builders a more specific way to evaluate long-context systems.","published_at":"2026-11-03","updated_at":"2026-11-07","record_date":"2026-11-03","date_kind":"published_at","topics":["claude","gemini"],"entities":["Ai2","StateBench","Olmo 3","Olmo Hybrid","Gated DeltaNet","Gemini 3.8 Flash","Claude Opus 5.5","GPT-6 Sol"],"related_patterns":["company-memory-needs-write-loops"],"novelty_label":"same_event","prior_refs":["https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-afffa2/"],"source_url":"https://huggingface.co/blog/allenai/statebench","source_urls":["https://huggingface.co/blog/allenai/statebench"],"schema_version":"newruntime-agent-readable-v0.2","stable_id":"post:statebench-testing-how-language-models-track-state-changes-f7c04b","status":"published","visuals":[],"editor":{"role":"editor","name":"Andrey Reshetnikov","profile":"https://newruntime.com/owner-profile.md","profile_json":"https://newruntime.com/owner-profile.json","same_as":["https://www.linkedin.com/in/reshetnikov1/","https://t.me/lapetuse"]},"routes":{"html":"https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-f7c04b/","markdown":"https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-f7c04b.md","json":"https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-f7c04b.json"},"markdown_url":"https://newruntime.com/posts/statebench-testing-how-language-models-track-state-changes-f7c04b.md","next_reads":[{"type":"topic","path":"/topics/claude/","url":"https://newruntime.com/topics/claude/","title":"Claude","reason":"Explore the claude topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/claude.md","json_url":"https://newruntime.com/topics/claude.json"},{"type":"topic","path":"/topics/gemini/","url":"https://newruntime.com/topics/gemini/","title":"Gemini","reason":"Explore the gemini topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/gemini.md","json_url":"https://newruntime.com/topics/gemini.json"},{"type":"collection","path":"/posts/","url":"https://newruntime.com/posts/","title":"Field Notes","reason":"Open the latest editorial analysis.","media_type":"text/html","json_url":"https://newruntime.com/posts.json"},{"type":"collection","path":"/sources/","url":"https://newruntime.com/sources/","title":"Sources","reason":"See the original sources this archive cites.","media_type":"text/html"}]}