{
  "id": "agent-reliability-glossary",
  "type": "glossary",
  "name": "Agent reliability glossary",
  "summary": "Working definitions for the terms this instance uses everywhere: eval, harness, canary, red-teaming, regression gate, pass^k, fault injection, sandbox. Each definition is bound to a source that uses the term operationally, so the glossary states how the field works rather than how a dictionary reads.",
  "locale": "en",
  "tags": [
    "glossary",
    "terminology",
    "evaluation"
  ],
  "relations": [
    {
      "rel": "related",
      "target": "deterministic-vs-probabilistic-evaluation"
    }
  ],
  "questions": [
    "What is an eval?",
    "What is an evaluation harness?",
    "What is a canary in agent operations?",
    "What is red-teaming for AI agents?",
    "What is a regression gate?",
    "What does pass^k mean?",
    "What is fault injection?",
    "What is a sandbox in agent evaluation?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "An eval is a structured, repeatable test that measures an LLM or LLM-based system against a defined dimension; frameworks package evals as registries of reusable templates.",
      "sources": [
        "openai-evals"
      ],
      "confidence": 0.9
    },
    {
      "id": "c2",
      "text": "An evaluation harness is the fixed machinery that presents tasks, captures behavior and computes verdicts; it is part of the measurement — formatting details alone have shifted benchmark scores by about 5%.",
      "sources": [
        "anthropic-eval-challenges"
      ],
      "confidence": 0.85
    },
    {
      "id": "c3",
      "text": "A canary is a continuously-run probe with a known-good baseline whose deviation signals trouble early — the operational descendant of the chaos-engineering steady-state hypothesis.",
      "sources": [
        "principles-chaos"
      ],
      "confidence": 0.85
    },
    {
      "id": "c4",
      "text": "Red-teaming is adversarial evaluation: probing a system for harmful or dangerous capability — persuasion, cyber offense, self-proliferation — under hostile framing instead of benign-path testing.",
      "sources": [
        "deepmind-dangerous-capabilities"
      ],
      "confidence": 0.85
    },
    {
      "id": "c5",
      "text": "A regression gate is an automated check that blocks a change when previously-verified behavior stops holding; eval frameworks exist largely to detect how model or system versions change application behavior.",
      "sources": [
        "openai-evals"
      ],
      "confidence": 0.8
    },
    {
      "id": "c6",
      "text": "pass^k is the probability that a system succeeds on all k repeated trials of the same task — a reliability metric that separates consistency from single-attempt capability, introduced with tau-bench.",
      "sources": [
        "tau-bench"
      ],
      "confidence": 0.9
    },
    {
      "id": "c7",
      "text": "Fault injection is deliberately introducing realistic disruptions — server failures, network outages, degraded dependencies — to verify a system's behavior against a steady-state baseline, per the chaos engineering method.",
      "sources": [
        "principles-chaos"
      ],
      "confidence": 0.85
    },
    {
      "id": "c8",
      "text": "A sandbox is a contained execution environment for untrusted code and agent actions; evaluation frameworks implement it with backends such as Docker and Kubernetes so tested behavior cannot reach production systems.",
      "sources": [
        "inspect-ai"
      ],
      "confidence": 0.85
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "industry_observation",
    "source_types": [
      "industry_observation",
      "paper"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Definitions carry an operational citation each — the reader can follow every term to a system that uses it, which generic AI glossaries do not provide.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/agent-reliability-glossary",
  "api_url": "https://agentreliability.dev/api/k/agent-reliability-glossary.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "Agent reliability glossary",
    "description": "Working definitions for the terms this instance uses everywhere: eval, harness, canary, red-teaming, regression gate, pass^k, fault injection, sandbox. Each definition is bound to a source that uses the term operationally, so the glossary states how the field works rather than how a dictionary reads.",
    "url": "https://agentreliability.dev/k/agent-reliability-glossary",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "Challenges in evaluating AI systems",
        "url": "https://www.anthropic.com/news/evaluating-ai-systems"
      },
      {
        "@type": "CreativeWork",
        "name": "openai/evals — framework for evaluating LLMs and LLM systems",
        "url": "https://github.com/openai/evals"
      },
      {
        "@type": "CreativeWork",
        "name": "Inspect — evaluation framework for large language models",
        "url": "https://inspect.aisi.org.uk/"
      },
      {
        "@type": "CreativeWork",
        "name": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
        "url": "https://arxiv.org/abs/2406.12045"
      },
      {
        "@type": "CreativeWork",
        "name": "Evaluating Frontier Models for Dangerous Capabilities",
        "url": "https://arxiv.org/abs/2403.13793"
      },
      {
        "@type": "CreativeWork",
        "name": "Principles of Chaos Engineering",
        "url": "https://principlesofchaos.org/"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "DefinedTerm"
  }
}