{
  "id": "evaluating-rag-in-agents",
  "type": "entity",
  "name": "Evaluating retrieval (RAG) inside agents",
  "summary": "The retrieval layer of an agent fails in three distinct ways — wrong passages retrieved, right passages used unfaithfully, fluent answer ungrounded in either — and each needs its own metric. Reference-free RAG evaluation makes this measurable without annotated ground truth, at the price of inheriting LLM-judge biases. In an agent, retrieved content is also an attack surface, so RAG evaluation and injection testing share fixtures.",
  "locale": "en",
  "tags": [
    "rag",
    "retrieval",
    "evaluation",
    "faithfulness",
    "grounding"
  ],
  "relations": [
    {
      "rel": "related",
      "target": "llm-as-judge"
    },
    {
      "rel": "related",
      "target": "prompt-injection-testing"
    },
    {
      "rel": "related",
      "target": "deterministic-vs-probabilistic-evaluation"
    }
  ],
  "questions": [
    "How do I evaluate the RAG pipeline inside my agent?",
    "What is faithfulness in RAG evaluation?",
    "Can I evaluate retrieval without labeled ground truth?"
  ],
  "claims": [
    {
      "id": "c1",
      "text": "Reference-free RAG evaluation exists and is dimensional: Ragas measures the retrieval system's capacity to find relevant, focused context passages, the model's faithful use of those passages, and generation quality — all without ground-truth human annotations.",
      "sources": [
        "ragas-paper"
      ],
      "confidence": 0.9
    },
    {
      "id": "c2",
      "text": "Reference-free metrics are model-graded, so they inherit the documented failure modes of LLM judges — position, verbosity and self-enhancement biases with limited reasoning on hard cases — and need the same calibration against human-audited samples.",
      "sources": [
        "mt-bench-llm-judge"
      ],
      "confidence": 0.85
    },
    {
      "id": "c3",
      "text": "In a tool-using agent, retrieved content is untrusted input: prompt injection through third-party content is the top-ranked LLM application risk, so the retrieval fixtures used for quality evaluation double as the seeding channel for injection tests.",
      "sources": [
        "owasp-llm-top10"
      ],
      "confidence": 0.85
    }
  ],
  "takeaways": [],
  "faqs": [],
  "evidence_tier": "secondary",
  "evidence": {
    "level": "industry_observation",
    "source_types": [
      "paper",
      "industry_observation"
    ]
  },
  "moat_flag": false,
  "winning_edge": "Treats RAG evaluation as three separable failure modes with an honest bias caveat on reference-free grading, and connects the quality fixtures to the security fixtures — a link the RAG-eval literature and the security literature each omit.",
  "confidence": 0.85,
  "last_verified": "2026-08-08",
  "canonical_url": "https://agentreliability.dev/k/evaluating-rag-in-agents",
  "api_url": "https://agentreliability.dev/api/k/evaluating-rag-in-agents.json",
  "jsonld": {
    "@context": "https://schema.org",
    "name": "Evaluating retrieval (RAG) inside agents",
    "description": "The retrieval layer of an agent fails in three distinct ways — wrong passages retrieved, right passages used unfaithfully, fluent answer ungrounded in either — and each needs its own metric. Reference-free RAG evaluation makes this measurable without annotated ground truth, at the price of inheriting LLM-judge biases. In an agent, retrieved content is also an attack surface, so RAG evaluation and injection testing share fixtures.",
    "url": "https://agentreliability.dev/k/evaluating-rag-in-agents",
    "license": "https://spdx.org/licenses/CC-BY-4.0.html",
    "dateModified": "2026-08-08",
    "citation": [
      {
        "@type": "CreativeWork",
        "name": "OWASP Top 10 for LLM Applications and Generative AI (2025)",
        "url": "https://genai.owasp.org/llm-top-10/"
      },
      {
        "@type": "CreativeWork",
        "name": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
        "url": "https://arxiv.org/abs/2306.05685"
      },
      {
        "@type": "CreativeWork",
        "name": "Ragas: Automated Evaluation of Retrieval Augmented Generation",
        "url": "https://arxiv.org/abs/2309.15217"
      }
    ],
    "author": {
      "@type": "Person",
      "name": "Santiago Santa María Morales",
      "jobTitle": "practitioner — harness engineering and agent evaluation in production"
    },
    "@type": "Article",
    "headline": "Evaluating retrieval (RAG) inside agents"
  }
}