{
  "id": "swe-bench-verified",
  "type": "entity",
  "name": "SWE-bench Verified",
  "summary": "The human-validated subset of SWE-bench: 500 tasks filtered from the original test set for quality, graded by unit-test verification against post-PR behavior. Its existence is the interesting fact — a flagship benchmark needed a human audit of its own tasks, which makes \"who benchmarks the benchmark?\" a standing question for anyone gating decisions on eval numbers.",
  "locale": "en",
  "tags": [
    "swe-bench",
    "benchmarks",
    "data-quality",
    "evaluation",
    "verification"
  ],
  "relations": [
    {
      "rel": "refines",
      "target": "measuring-agent-reliability"
    },
    {
      "rel": "related",
      "target": "public-benchmarks-vs-private-task-suites"
    },
    {
      "rel": "related",
      "target": "opaque-rotating-test-sets"
    }
  ],
  "questions": [
    "What is SWE-bench Verified?",
    "Why did SWE-bench need a verified subset?",
    "Can benchmark tasks themselves be wrong?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "SWE-bench Verified is a subset of 500 samples from the SWE-bench test set that have been human-validated for quality.",
      "sources": [
        "swebench-verified-dataset"
      ],
      "confidence": 0.9
    },
    {
      "id": "c2",
      "text": "Verified keeps execution-based grading: task resolution is checked by unit-test verification using post-PR behavior as the reference solution.",
      "sources": [
        "swebench-verified-dataset"
      ],
      "confidence": 0.9
    },
    {
      "id": "c3",
      "text": "The parent benchmark it filters is large and real — 2,294 GitHub issues from Python repositories requiring multi-file edits and execution-environment interaction — which is exactly the scale at which unaudited task quality becomes a measurement risk.",
      "sources": [
        "swe-bench"
      ],
      "confidence": 0.9
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "benchmark",
    "source_types": [
      "benchmark",
      "paper"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Reads Verified as evidence about benchmarks in general — task quality is a measured variable, not an assumption — rather than as one more leaderboard entry, and connects that lesson to private-suite hygiene.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/swe-bench-verified",
  "api_url": "https://agentreliability.dev/api/k/swe-bench-verified.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "SWE-bench Verified",
    "description": "The human-validated subset of SWE-bench: 500 tasks filtered from the original test set for quality, graded by unit-test verification against post-PR behavior. Its existence is the interesting fact — a flagship benchmark needed a human audit of its own tasks, which makes \"who benchmarks the benchmark?\" a standing question for anyone gating decisions on eval numbers.",
    "url": "https://agentreliability.dev/k/swe-bench-verified",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "SWE-bench Verified dataset card (princeton-nlp/SWE-bench_Verified)",
        "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified"
      },
      {
        "@type": "CreativeWork",
        "name": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
        "url": "https://arxiv.org/abs/2310.06770"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "SWE-bench Verified"
  }
}