Agent Evaluation

from agentevals.trajectory.match import create_trajectory_match_evaluator
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage

must_file_issue = create_trajectory_match_evaluator(
    trajectory_match_mode="superset",       # the agent may do more, but must do at least this
    tool_args_match_mode="ignore",
)

reference = [
    HumanMessage("The memory page is wrong -- report it."),
    AIMessage("", tool_calls=[{"name": "github_open_issue", "args": {}, "id": "ref-1"}]),
    ToolMessage("filed", tool_call_id="ref-1"),
    AIMessage("Filed."),
]

result = agent.invoke({"messages": [("user", "The memory page is wrong -- report it.")]})
must_file_issue(outputs=result["messages"], reference_outputs=reference)
# -> {"key": "trajectory_superset_match", "score": True, ...}

import json
from dataclasses import dataclass, field

from langchain_ollama import ChatOllama
from pydantic import BaseModel


@dataclass
class Case:
    question: str
    expected_tools: list[str]                     # in the order a good trajectory calls them
    expected_pages: list[str]                     # pages a correct answer must cite
    forbidden_tools: list[str] = field(default_factory=list)


DATASET = [
    Case("What does ef_search control in HNSW?", ["docs_search"],
         ["modules/ROOT/pages/database/vector-rag/ann-indexes.adoc"]),
    Case("The memory page says MessageWindowChatMemory keeps 10 messages -- report it.",
         ["docs_search", "github_open_issue"], ["modules/ROOT/pages/ai/agents/memory.adoc"]),
    Case("What's the weather in Madrid?", [], [], forbidden_tools=["github_open_issue"]),
]


def tool_calls(messages) -> list[dict]:
    return [{"name": c["name"], "args": c["args"]}
            for m in messages if m.type == "ai" for c in (m.tool_calls or [])]


def tool_metrics(case: Case, calls: list[dict]) -> dict:
    called = [c["name"] for c in calls]
    expected = set(case.expected_tools)
    hits = len(expected & set(called))
    remaining = iter(called)                      # subsequence: repeats and extra calls are allowed
    return {
        "tool_recall": hits / len(expected) if expected else 1.0,
        "tool_precision": hits / len(set(called)) if called else float(not expected),
        "in_order": all(t in remaining for t in case.expected_tools),
        "forbidden_called": any(n in case.forbidden_tools for n in called),
        "steps": len(calls),
    }


class Verdict(BaseModel):
    grounded: int      # 1-5: every claim supported by a cited page
    complete: int      # 1-5: answers the whole question
    cites_expected: bool
    reasoning: str


judge = ChatOllama(model="llama3.1:8b", temperature=0).with_structured_output(Verdict)
RUBRIC = """Grade the answer. grounded: 5 = every claim is supported by a cited page, 1 = mostly
unsupported. complete: 5 = fully answers the question, 1 = does not. cites_expected: does it cite
all of these pages: {pages}? Explain briefly in reasoning before scoring.
Question: {question}
Answer: {answer}"""


def evaluate(agent, dataset=DATASET) -> list[dict]:
    rows = []
    for case in dataset:
        result = agent.invoke({"messages": [("user", case.question)]})
        messages = result["messages"]
        verdict = judge.invoke(RUBRIC.format(pages=case.expected_pages, question=case.question,
                                             answer=messages[-1].content))
        rows.append({"question": case.question, **tool_metrics(case, tool_calls(messages)),
                     **verdict.model_dump(exclude={"reasoning"})})
    return rows

import statistics

import pytest

from docsassistant.agent import build_agent            # the agent under test, with stubbed write tools
from eval_harness import DATASET, evaluate


@pytest.fixture(scope="session")
def results():
    return evaluate(build_agent(dry_run=True))         # dry_run: github_open_issue records, never posts


def test_never_calls_forbidden_tools(results):
    assert not any(r["forbidden_called"] for r in results)


def test_tool_selection(results):
    assert statistics.mean(r["tool_recall"] for r in results) >= 0.9


def test_groundedness(results):
    assert statistics.mean(r["grounded"] for r in results) >= 4.0


def test_efficiency(results):
    assert max(r["steps"] for r in results) <= 6

import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
import org.springframework.ai.document.Document;
import org.springframework.ai.evaluation.EvaluationRequest;
import org.springframework.ai.evaluation.EvaluationResponse;

@SpringBootTest
class DocsAssistantEvaluationTest {

    @Autowired DocsAssistantAgent agent;
    @Autowired ChatClient.Builder judgeBuilder;          // configured with the judge model

    @Test
    void answerIsSupportedByTheDocs() {
        String answer = agent.ask("What does ef_search control in HNSW?");
        List<Document> sources = agent.lastRetrievedDocuments();

        EvaluationResponse verdict = FactCheckingEvaluator.builder(judgeBuilder).build()
                .evaluate(new EvaluationRequest(sources, answer));

        assertThat(verdict.isPass()).as(verdict.getFeedback()).isTrue();
    }
}

record Verdict(int grounded, boolean citesExpected, String reasoning) {}

interface Judge {
    @UserMessage("""
            Grade the answer. grounded: 5 = every claim supported by a cited page, 1 = mostly unsupported.
            citesExpected: does it cite {{page}}? Explain in reasoning first.
            Question: {{question}}
            Answer: {{answer}}""")
    Verdict grade(@V("question") String question, @V("answer") String answer, @V("page") String page);
}

class DocsAssistantEvalTest {

    Judge judge = AiServices.create(Judge.class, judgeModel);

    @Test
    void efSearchAnswerIsGrounded() {
        String answer = assistant.chat("eval-user", "What does ef_search control in HNSW?");
        Verdict v = judge.grade("What does ef_search control in HNSW?", answer,
                "modules/ROOT/pages/database/vector-rag/ann-indexes.adoc");
        assertThat(v.grounded()).as(v.reasoning()).isGreaterThanOrEqualTo(4);
        assertThat(v.citesExpected()).isTrue();
    }
}