{
  "id": "process-vs-outcome-evaluation",
  "type": "entity",
  "name": "Process vs outcome evaluation of agent trajectories",
  "summary": "Two places to point the grader: the final state (outcome) or every intermediate step (process). Outcome grading is cheap, objective and gameable by lucky trajectories; step-level grading catches reasoning that arrives at the right answer wrong, at the cost of expensive labels. The research result: process supervision beat outcome supervision decisively on hard math. For agents, outcome grading gates releases while process signals explain failures and catch right-for-the-wrong-reason passes.",
  "locale": "en",
  "tags": [
    "process-supervision",
    "trajectories",
    "evaluation",
    "grading",
    "reasoning"
  ],
  "relations": [
    {
      "rel": "related",
      "target": "deterministic-vs-probabilistic-evaluation"
    },
    {
      "rel": "related",
      "target": "llm-as-judge"
    },
    {
      "rel": "refines",
      "target": "measuring-agent-reliability"
    }
  ],
  "questions": [
    "Should I grade my agent's steps or only its final result?",
    "What is process supervision?",
    "Why did my agent pass the eval with broken reasoning?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "Process supervision — feedback on each intermediate reasoning step — significantly outperforms outcome supervision on the MATH dataset, with the process-supervised reward model reaching 78% on a representative test subset.",
      "sources": [
        "verify-step-by-step"
      ],
      "confidence": 0.9
    },
    {
      "id": "c2",
      "text": "Step-level evaluation has a public data foundation: PRM800K releases 800,000 step-level human feedback labels used to train the best process reward model.",
      "sources": [
        "verify-step-by-step"
      ],
      "confidence": 0.9
    },
    {
      "id": "c3",
      "text": "Outcome grading remains the production gate for tool-using agents: comparing final system state against an annotated goal is reproducible and narration-independent, which is what release decisions need.",
      "sources": [
        "tau-bench"
      ],
      "confidence": 0.9
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "benchmark",
    "source_types": [
      "paper",
      "benchmark"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Assigns each grading mode a production role — outcome gates, process explains — instead of importing the research debate wholesale, and names the failure mode (right answer, wrong reasoning, lucky pass) that only step-level signals catch.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/process-vs-outcome-evaluation",
  "api_url": "https://agentreliability.dev/api/k/process-vs-outcome-evaluation.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "Process vs outcome evaluation of agent trajectories",
    "description": "Two places to point the grader: the final state (outcome) or every intermediate step (process). Outcome grading is cheap, objective and gameable by lucky trajectories; step-level grading catches reasoning that arrives at the right answer wrong, at the cost of expensive labels. The research result: process supervision beat outcome supervision decisively on hard math. For agents, outcome grading gates releases while process signals explain failures and catch right-for-the-wrong-reason passes.",
    "url": "https://agentreliability.dev/k/process-vs-outcome-evaluation",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
        "url": "https://arxiv.org/abs/2406.12045"
      },
      {
        "@type": "CreativeWork",
        "name": "Let's Verify Step by Step",
        "url": "https://arxiv.org/abs/2305.20050"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "Process vs outcome evaluation of agent trajectories"
  }
}