{
  "id": "deterministic-vs-probabilistic-evaluation",
  "type": "entity",
  "name": "Deterministic vs probabilistic evaluation",
  "summary": "Two grading regimes for agent behavior. Deterministic evaluation checks outcomes mechanically — exact state comparison, executable tests — and reproduces run to run. Probabilistic evaluation delegates judgment to a model or human grader, trading reproducibility for coverage of open-ended outputs. Production reliability work starts deterministic and adds probabilistic grading only where mechanical checks cannot reach.",
  "locale": "en",
  "tags": [
    "evaluation",
    "harness",
    "reproducibility",
    "llm-as-judge"
  ],
  "relations": [
    {
      "rel": "compared_in",
      "target": "deterministic-harnesses-vs-llm-as-judge"
    },
    {
      "rel": "related",
      "target": "llm-as-judge"
    }
  ],
  "questions": [
    "What is the difference between deterministic and probabilistic evaluation?",
    "When should I trust an exact check over a model grader?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "tau-bench grades agents deterministically by comparing the final database state of each episode against an annotated goal state, which catches wrong actions independently of how plausible the transcript looks.",
      "sources": [
        "tau-bench"
      ],
      "confidence": 0.95
    },
    {
      "id": "c2",
      "text": "Probabilistic grading with an LLM judge reaches over 80% agreement with human preferences on MT-Bench and Chatbot Arena, but carries documented position, verbosity and self-enhancement biases.",
      "sources": [
        "mt-bench-llm-judge"
      ],
      "confidence": 0.9
    },
    {
      "id": "c3",
      "text": "Even nominally objective benchmarks are fragile measurements: Anthropic reports that formatting inconsistencies alone moved MMLU scores by about 5%.",
      "sources": [
        "anthropic-eval-challenges"
      ],
      "confidence": 0.9
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "industry_observation",
    "source_types": [
      "paper",
      "industry_observation"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Names the trade-off explicitly and routes the reader to a decision rule (the harness-vs-judge comparison) instead of describing each method in isolation, which is how the topic is usually covered.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/deterministic-vs-probabilistic-evaluation",
  "api_url": "https://agentreliability.dev/api/k/deterministic-vs-probabilistic-evaluation.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "Deterministic vs probabilistic evaluation",
    "description": "Two grading regimes for agent behavior. Deterministic evaluation checks outcomes mechanically — exact state comparison, executable tests — and reproduces run to run. Probabilistic evaluation delegates judgment to a model or human grader, trading reproducibility for coverage of open-ended outputs. Production reliability work starts deterministic and adds probabilistic grading only where mechanical checks cannot reach.",
    "url": "https://agentreliability.dev/k/deterministic-vs-probabilistic-evaluation",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "Challenges in evaluating AI systems",
        "url": "https://www.anthropic.com/news/evaluating-ai-systems"
      },
      {
        "@type": "CreativeWork",
        "name": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
        "url": "https://arxiv.org/abs/2306.05685"
      },
      {
        "@type": "CreativeWork",
        "name": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
        "url": "https://arxiv.org/abs/2406.12045"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "Deterministic vs probabilistic evaluation"
  }
}