{
  "id": "agentbench",
  "type": "entity",
  "name": "AgentBench",
  "summary": "A multi-environment benchmark evaluating LLMs as agents across eight distinct settings (operating system, database, knowledge graph, games, web tasks and more). Its durable findings: commercial frontier models act competently as agents while sub-70B open models lag sharply, and the failures concentrate in long-horizon reasoning, decision-making and instruction following — not in single-turn knowledge.",
  "locale": "en",
  "tags": [
    "agentbench",
    "benchmarks",
    "agents",
    "evaluation"
  ],
  "relations": [
    {
      "rel": "related",
      "target": "measuring-agent-reliability"
    },
    {
      "rel": "related",
      "target": "webarena"
    },
    {
      "rel": "related",
      "target": "gaia-benchmark"
    }
  ],
  "questions": [
    "What is AgentBench?",
    "Where do LLM agents fail according to AgentBench?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "AgentBench evaluates LLMs as autonomous agents across eight distinct interactive environments rather than a single task family.",
      "sources": [
        "agentbench-paper"
      ],
      "confidence": 0.95
    },
    {
      "id": "c2",
      "text": "AgentBench found top commercial LLMs show strong agent ability in complex environments while open-source models up to 70B trail by a significant margin, with key obstacles in long-term reasoning, decision-making and instruction following.",
      "sources": [
        "agentbench-paper"
      ],
      "confidence": 0.9
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "benchmark",
    "source_types": [
      "benchmark",
      "paper"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Extracts the failure-mode finding (long-horizon reasoning, not knowledge) that predicts production agent behavior, instead of re-listing the eight environments as trivia.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/agentbench",
  "api_url": "https://agentreliability.dev/api/k/agentbench.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "AgentBench",
    "description": "A multi-environment benchmark evaluating LLMs as agents across eight distinct settings (operating system, database, knowledge graph, games, web tasks and more). Its durable findings: commercial frontier models act competently as agents while sub-70B open models lag sharply, and the failures concentrate in long-horizon reasoning, decision-making and instruction following — not in single-turn knowledge.",
    "url": "https://agentreliability.dev/k/agentbench",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "AgentBench: Evaluating LLMs as Agents",
        "url": "https://arxiv.org/abs/2308.03688"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "AgentBench"
  }
}