{"type":"post","id":"automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4","slug":"automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4","title":"AutomationBench fixes replaced prose request bodies with JSON Schemas; Opus 5.5 hit 92.15% partial, 70.83% strict.","description":"AutomationBench-PI documents an audit that changes how its agents see tools and how several business-workflow tasks are graded.","retrieval_nugget":"AutomationBench-PI documents an audit that changes how its agents see tools and how several business-workflow tasks are graded. The repository reports Claude Opus 5.5 at 92.15% on partial scoring and 70.83% on strict scoring. For teams evaluating agents, the episode shows how interface and verifier choices can change benchmark results.","published_at":"2026-11-03","updated_at":"2026-11-03","record_date":"2026-11-03","date_kind":"published_at","topics":["claude"],"entities":["AutomationBench-PI","AutomationBench","Claude Opus 5.5","Zapier","vLLM","SGLang","Jira","Monday"],"related_patterns":["verification-bandwidth-is-the-scarce-resource"],"novelty_label":"new_fact","what_changed":"AutomationBench-PI reports verifier fixes replacing prose request bodies with JSON Schemas and revised Opus 5.5 scores of 92.15% partial and 70.83% strict.","prior_refs":[],"source_url":"https://github.com/xeophon/AutomationBench-PI","source_urls":["https://github.com/xeophon/AutomationBench-PI"],"schema_version":"newruntime-agent-readable-v0.2","stable_id":"post:automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4","status":"published","visuals":[],"editor":{"role":"editor","name":"Andrey Reshetnikov","profile":"https://newruntime.com/owner-profile.md","profile_json":"https://newruntime.com/owner-profile.json","same_as":["https://www.linkedin.com/in/reshetnikov1/","https://t.me/lapetuse"]},"routes":{"html":"https://newruntime.com/posts/automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4/","markdown":"https://newruntime.com/posts/automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4.md","json":"https://newruntime.com/posts/automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4.json"},"markdown_url":"https://newruntime.com/posts/automationbench-fixes-replaced-prose-request-bodies-with-json-7f93b4.md","next_reads":[{"type":"topic","path":"/topics/claude/","url":"https://newruntime.com/topics/claude/","title":"Claude","reason":"Explore the claude topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/claude.md","json_url":"https://newruntime.com/topics/claude.json"},{"type":"collection","path":"/posts/","url":"https://newruntime.com/posts/","title":"Field Notes","reason":"Open the latest editorial analysis.","media_type":"text/html","json_url":"https://newruntime.com/posts.json"},{"type":"collection","path":"/sources/","url":"https://newruntime.com/sources/","title":"Sources","reason":"See the original sources this archive cites.","media_type":"text/html"}]}