{
  "schema_version": "newruntime-topic-hub-v0.1",
  "type": "topic_hub",
  "slug": "evals",
  "title": "Agent evals - New Runtime",
  "description": "A record of evals moving from model benchmarks toward behavior, workflow completion, skill reliability, and review burden.",
  "answer": [
    "Agent evals need to measure behavior in a workflow, not only answer quality in a prompt.",
    "The relevant unit is completed work under constraints: tools, files, permissions, time, cost, and review outcome.",
    "A good eval makes regressions visible before the agent is trusted with larger work."
  ],
  "search_intents": [
    "agent evals",
    "AI agent evaluation",
    "LLM workflow benchmarks"
  ],
  "status": "featured",
  "last_updated": "2026-07-16",
  "counts": {
    "total": 13,
    "signals": 11,
    "patterns": 2,
    "posts": 0,
    "atlas": 0,
    "sources": 22
  },
  "routes": {
    "html": "https://newruntime.com/topics/evals/",
    "markdown": "https://newruntime.com/topics/evals.md",
    "json": "https://newruntime.com/topics/evals.json"
  },
  "top_sources": [
    "https://addyo.substack.com/p/own-the-outer-loop",
    "https://addyosmani.com/blog/new-sdlc-vibe-coding",
    "https://anthropic.com/research/AI-assistance-coding-skills",
    "https://arxiv.org/abs/2602.12670",
    "https://arxiv.org/abs/2603.29919",
    "https://arxiv.org/abs/2605.24050",
    "https://databricks.com/blog/benchmarking-coding-agents-databricks-multi-million-line-codebase",
    "https://dbreunig.com/2026/06/22/the-problem-is-prompt-debt.html",
    "https://developers.openai.com/api/docs/guides/latest-model",
    "https://github.com/benchflow-ai/skillsbench",
    "https://github.com/OpenDataBox/MemoryData",
    "https://github.com/shadcn/improve"
  ],
  "patterns": [
    {
      "kind": "Pattern",
      "slug": "skills-become-portable-capability-layer",
      "title": "Skills become a portable capability layer",
      "description": "Agent skills are emerging as a portable capability layer, but their value depends on progressive disclosure, provenance, security review, and behavioral evaluation.",
      "date": "2026-07-16",
      "topics": [
        "agent-skills",
        "evals",
        "context-engineering"
      ],
      "source_count": 4,
      "metric": "medium",
      "url": "https://newruntime.com/patterns/skills-become-portable-capability-layer/"
    },
    {
      "kind": "Pattern",
      "slug": "verification-bandwidth-is-the-scarce-resource",
      "title": "Verification bandwidth is the scarce engineering resource",
      "description": "The primary bottleneck in agentic software delivery is moving from code production to the human and machine capacity required to verify it.",
      "date": "2026-07-16",
      "topics": [
        "coding-agents",
        "evals",
        "engineering-management"
      ],
      "source_count": 5,
      "metric": "high",
      "url": "https://newruntime.com/patterns/verification-bandwidth-is-the-scarce-resource/"
    }
  ],
  "field_notes": [],
  "raw_signals": [
    {
      "kind": "Raw Signal",
      "slug": "agent-skills-need-behavioral-evals",
      "title": "Agent skills need behavioral evals, not prose review",
      "description": "Benchmarks show that expert-authored skills can help while self-generated skills can underperform a no-skill baseline.",
      "date": "2026-07-16",
      "topics": [
        "agent-skills",
        "evals",
        "context-engineering"
      ],
      "source_count": 5,
      "metric": "structural",
      "url": "https://newruntime.com/signals/agent-skills-need-behavioral-evals/"
    },
    {
      "kind": "Raw Signal",
      "slug": "verification-capacity-coding-agent-bottleneck",
      "title": "Verification capacity becomes the coding-agent bottleneck",
      "description": "Coding agents can parallelize implementation faster than engineering teams can expand review judgment.",
      "date": "2026-07-16",
      "topics": [
        "coding-agents",
        "evals",
        "agent-ops",
        "engineering-management"
      ],
      "source_count": 4,
      "metric": "structural",
      "url": "https://newruntime.com/signals/verification-capacity-coding-agent-bottleneck/"
    },
    {
      "kind": "Raw Signal",
      "slug": "ai-feedback-needs-behavioral-traces",
      "title": "AI product feedback needs behavioral traces",
      "description": "For nondeterministic products, edits, retries, overrides, abandonment, and sampled outputs reveal more than isolated ratings.",
      "date": "2026-07-14",
      "topics": [
        "product-feedback",
        "evals",
        "human-traces"
      ],
      "source_count": 1,
      "metric": "notable",
      "url": "https://newruntime.com/signals/ai-feedback-needs-behavioral-traces/"
    },
    {
      "kind": "Raw Signal",
      "slug": "databricks-benchmarks-real-coding-agent-economics",
      "title": "Databricks benchmarks coding agents on its own codebase",
      "description": "Databricks evaluates agents on fresh internal pull-request tasks and measures success alongside runtime, tokens, and cost.",
      "date": "2026-07-12",
      "topics": [
        "coding-agents",
        "evals",
        "agent-economics"
      ],
      "source_count": 1,
      "metric": "structural",
      "url": "https://newruntime.com/signals/databricks-benchmarks-real-coding-agent-economics/"
    },
    {
      "kind": "Raw Signal",
      "slug": "gpt-5-6-shorter-prompts",
      "title": "Newer agent models can reward shorter prompts",
      "description": "OpenAI's migration guidance emphasizes concise instructions, explicit contracts, and less compensating prose for newer models.",
      "date": "2026-07-10",
      "topics": [
        "prompt-engineering",
        "model-migration",
        "evals"
      ],
      "source_count": 1,
      "metric": "incremental",
      "url": "https://newruntime.com/signals/gpt-5-6-shorter-prompts/"
    },
    {
      "kind": "Raw Signal",
      "slug": "continuous-evals-for-multi-agent-systems",
      "title": "Multi-agent systems need continuous eval pipelines",
      "description": "A Google workflow evaluates not only the final answer but also routing, delegation, tool calls, and the trajectory between agents.",
      "date": "2026-07-07",
      "topics": [
        "multi-agent",
        "evals",
        "observability"
      ],
      "source_count": 1,
      "metric": "notable",
      "url": "https://newruntime.com/signals/continuous-evals-for-multi-agent-systems/"
    },
    {
      "kind": "Raw Signal",
      "slug": "agent-memory-evaluated-as-system-behavior",
      "title": "Agent memory is being evaluated as system behavior",
      "description": "MemoryData evaluates what an agent stores, retrieves, updates, and forgets across a sequence instead of grading one final answer.",
      "date": "2026-07-06",
      "topics": [
        "agent-memory",
        "evals",
        "knowledge-systems"
      ],
      "source_count": 2,
      "metric": "structural",
      "url": "https://newruntime.com/signals/agent-memory-evaluated-as-system-behavior/"
    },
    {
      "kind": "Raw Signal",
      "slug": "harvey-trains-legal-agent-with-applied-compute",
      "title": "Harvey trained a legal agent with an applied compute loop",
      "description": "Harvey describes domain experts, task environments, evaluation, and iterative training as one system for legal agent performance.",
      "date": "2026-07-03",
      "topics": [
        "vertical-agents",
        "evals",
        "legal-ai"
      ],
      "source_count": 1,
      "metric": "notable",
      "url": "https://newruntime.com/signals/harvey-trains-legal-agent-with-applied-compute/"
    },
    {
      "kind": "Raw Signal",
      "slug": "verifiability-is-an-ai-product-feature",
      "title": "Verifiability is an AI product feature",
      "description": "Hamel Husain's eval-smell framework treats missing traces, weak rubrics, and unverifiable outputs as product defects.",
      "date": "2026-07-02",
      "topics": [
        "evals",
        "observability",
        "ai-product"
      ],
      "source_count": 1,
      "metric": "structural",
      "url": "https://newruntime.com/signals/verifiability-is-an-ai-product-feature/"
    },
    {
      "kind": "Raw Signal",
      "slug": "prompt-debt-becomes-legacy-code",
      "title": "Prompt debt turns natural language into legacy code",
      "description": "System prompts accumulate patches for edge cases until model changes and local fixes make the behavior fragile and opaque.",
      "date": "2026-06-29",
      "topics": [
        "prompt-engineering",
        "evals",
        "technical-debt"
      ],
      "source_count": 1,
      "metric": "structural",
      "url": "https://newruntime.com/signals/prompt-debt-becomes-legacy-code/"
    },
    {
      "kind": "Raw Signal",
      "slug": "ai-compresses-coding-not-whole-sdlc",
      "title": "AI compresses coding, not the whole software lifecycle",
      "description": "Implementation time can collapse while requirements, architecture, validation, review, and maintenance stay constrained by human judgment.",
      "date": "2026-06-23",
      "topics": [
        "software-lifecycle",
        "coding-agents",
        "evals"
      ],
      "source_count": 2,
      "metric": "structural",
      "url": "https://newruntime.com/signals/ai-compresses-coding-not-whole-sdlc/"
    }
  ],
  "atlas_records": []
}