{"type":"analysis","id":"analysis:research-dab-is-a-real-messy-data-agent-eval-not-a-production-quality-oracle","slug":"research-dab-is-a-real-messy-data-agent-eval-not-a-production-quality-oracle","title":"DAB is a real messy-data agent eval, not a production-quality oracle","description":"UC Berkeley’s EPIC Data Lab and PromptQL released Data Agent Benchmark with per-query Python graders over multi-database tasks whose mess is deliberately induced. Untuned agents still miss most questions; tuned leaderboard entries score far higher, and no study shows DAB ranks production warehouse agents.","published_at":"2026-10-25T19:00:00.000Z","updated_at":"2026-10-25T19:00:00.000Z","record_date":"2026-10-25","topics":["agents","tools","claude","research"],"source_urls":["https://ucbepic.github.io/DataAgentBench/","https://raw.githubusercontent.com/ucbepic/DataAgentBench/main/README.md","https://huggingface.co/datasets/ruiyingm/DataAgentBench-data","https://arxiv.org/html/2603.20576v3","https://arxiv.org/html/2603.20576v1","https://raw.githubusercontent.com/ucbepic/DataAgentBench/main/SUBMISSION_RUBRIC.md","https://arxiv.org/abs/2603.20576","https://promptql.io/resources/dab-benchmark-study","https://hamel.dev/notes/llm/ai-product-engineering/evals-data-agents.html","https://hex.tech/blog/evaluate-data-agents/"],"language":"en","routes":{"html":"https://newruntime.com/analysis/research-dab-is-a-real-messy-data-agent-eval-not-a-production-quality-oracle/","markdown":"https://newruntime.com/analysis/research-dab-is-a-real-messy-data-agent-eval-not-a-production-quality-oracle.md","json":"https://newruntime.com/analysis/research-dab-is-a-real-messy-data-agent-eval-not-a-production-quality-oracle.json"},"author":{"role":"author","name":"Andrey Reshetnikov","profile":"https://newruntime.com/owner-profile.md","profile_json":"https://newruntime.com/owner-profile.json","same_as":["https://www.linkedin.com/in/reshetnikov1/","https://t.me/lapetuse"]},"editorial_provenance":{"desk":"authorial","target_ref":"deep_research:44b06a34-efaf-4b2d-ba12-7fefca06d7b8","authorial_article_id":"44b06a34-efaf-4b2d-ba12-7fefca06d7b8"},"next_reads":[{"type":"topic","path":"/topics/agents/","url":"https://newruntime.com/topics/agents/","title":"Agents","reason":"Explore the agents topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/agents.md","json_url":"https://newruntime.com/topics/agents.json"},{"type":"topic","path":"/topics/tools/","url":"https://newruntime.com/topics/tools/","title":"Tools","reason":"Explore the tools topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/tools.md","json_url":"https://newruntime.com/topics/tools.json"},{"type":"topic","path":"/topics/claude/","url":"https://newruntime.com/topics/claude/","title":"Claude","reason":"Explore the claude topic hub.","media_type":"text/html","markdown_url":"https://newruntime.com/topics/claude.md","json_url":"https://newruntime.com/topics/claude.json"},{"type":"collection","path":"/analysis/","url":"https://newruntime.com/analysis/","title":"Analysis","reason":"Open the latest authorial analysis.","media_type":"text/html","json_url":"https://newruntime.com/analysis.json"},{"type":"collection","path":"/source-ledger.json","url":"https://newruntime.com/source-ledger.json","title":"Source Ledger","reason":"Inspect the public source evidence graph.","media_type":"application/json"}]}