{
  "id": "llm-as-judge",
  "type": "entity",
  "name": "LLM-as-judge and its failure modes",
  "summary": "Using a strong language model to grade another model's output. Cheap, scalable, and surprisingly aligned with human preference on open-ended chat — but a biased instrument: judges favor certain answer positions, longer answers, and their own outputs, and their reasoning limits cap what they can grade. Usable in production only with its biases measured and bounded.",
  "locale": "en",
  "tags": [
    "llm-as-judge",
    "evaluation",
    "biases",
    "grading"
  ],
  "relations": [
    {
      "rel": "compared_in",
      "target": "deterministic-harnesses-vs-llm-as-judge"
    },
    {
      "rel": "related",
      "target": "deterministic-vs-probabilistic-evaluation"
    }
  ],
  "questions": [
    "Can I trust an LLM to grade my agent?",
    "What are the known biases of LLM judges?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "Strong LLM judges like GPT-4 reach over 80% agreement with human preferences on MT-Bench and Chatbot Arena — the same level as agreement between humans.",
      "sources": [
        "mt-bench-llm-judge"
      ],
      "confidence": 0.9
    },
    {
      "id": "c2",
      "text": "LLM judges exhibit position bias, verbosity bias and self-enhancement bias, plus limited reasoning ability on hard grading tasks; these are documented, measurable failure modes.",
      "sources": [
        "mt-bench-llm-judge"
      ],
      "confidence": 0.95
    },
    {
      "id": "c3",
      "text": "Model-graded evaluation is a first-class template in OpenAI's evals framework, alongside basic (deterministic) eval templates.",
      "sources": [
        "openai-evals"
      ],
      "confidence": 0.9
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "benchmark",
    "source_types": [
      "benchmark",
      "paper"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Presents the judge as an instrument with a calibration sheet — known biases, agreement ceiling, reasoning limits — rather than as a yes/no debate about whether LLM grading \"works\".",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/llm-as-judge",
  "api_url": "https://agentreliability.dev/api/k/llm-as-judge.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "LLM-as-judge and its failure modes",
    "description": "Using a strong language model to grade another model's output. Cheap, scalable, and surprisingly aligned with human preference on open-ended chat — but a biased instrument: judges favor certain answer positions, longer answers, and their own outputs, and their reasoning limits cap what they can grade. Usable in production only with its biases measured and bounded.",
    "url": "https://agentreliability.dev/k/llm-as-judge",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "openai/evals — framework for evaluating LLMs and LLM systems",
        "url": "https://github.com/openai/evals"
      },
      {
        "@type": "CreativeWork",
        "name": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
        "url": "https://arxiv.org/abs/2306.05685"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "LLM-as-judge and its failure modes"
  }
}