{
  "id": "statistical-rigor-in-evals",
  "type": "entity",
  "name": "Statistical rigor in agent evals",
  "summary": "Treating an eval score as a measurement with uncertainty instead of a fact: report error bars, model your task set as a sample from a larger population of possible tasks, and never call a two-point difference a regression (or an improvement) without knowing the noise floor. The cheapest upgrade available to most eval pipelines is a standard error next to every number.",
  "locale": "en",
  "tags": [
    "statistics",
    "evaluation",
    "error-bars",
    "variance",
    "metrics"
  ],
  "relations": [
    {
      "rel": "refines",
      "target": "measuring-agent-reliability"
    },
    {
      "rel": "related",
      "target": "deterministic-vs-probabilistic-evaluation"
    },
    {
      "rel": "related",
      "target": "goodhart-resistance"
    }
  ],
  "questions": [
    "Is a 2-point eval difference between two models real?",
    "How many eval runs do I need before trusting a comparison?",
    "Why do my eval scores move between identical runs?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "Eval questions should be conceptualized as draws from an unseen super-population, with results analyzed and reported using statistical formulas that minimize noise and maximize the informativeness of model comparisons.",
      "sources": [
        "error-bars-evals"
      ],
      "confidence": 0.9
    },
    {
      "id": "c2",
      "text": "Measurement noise in evals is empirically material: formatting inconsistencies alone have moved MMLU scores by about 5% — larger than many claimed model-to-model differences.",
      "sources": [
        "anthropic-eval-challenges"
      ],
      "confidence": 0.9
    },
    {
      "id": "c3",
      "text": "Repeated trials are the raw material of agent statistics: pass^k exists because single-run success and consistency across k runs diverge sharply in practice.",
      "sources": [
        "tau-bench"
      ],
      "confidence": 0.85
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "industry_observation",
    "source_types": [
      "paper",
      "industry_observation"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Turns \"add error bars\" from statistical advice into an agent-eval decision rule — compare the claimed effect against the known noise floor before gating anything — with the 5% formatting artifact as the memorable counterexample.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/statistical-rigor-in-evals",
  "api_url": "https://agentreliability.dev/api/k/statistical-rigor-in-evals.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "Statistical rigor in agent evals",
    "description": "Treating an eval score as a measurement with uncertainty instead of a fact: report error bars, model your task set as a sample from a larger population of possible tasks, and never call a two-point difference a regression (or an improvement) without knowing the noise floor. The cheapest upgrade available to most eval pipelines is a standard error next to every number.",
    "url": "https://agentreliability.dev/k/statistical-rigor-in-evals",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "Challenges in evaluating AI systems",
        "url": "https://www.anthropic.com/news/evaluating-ai-systems"
      },
      {
        "@type": "CreativeWork",
        "name": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
        "url": "https://arxiv.org/abs/2406.12045"
      },
      {
        "@type": "CreativeWork",
        "name": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
        "url": "https://arxiv.org/abs/2411.00640"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "Statistical rigor in agent evals"
  }
}