Evaluation at Scale

import os

from langchain.chat_models import init_chat_model
from langchain.embeddings import init_embeddings
from langchain_core.documents import Document
from ragas.testset import TestsetGenerator


def synthesize(chunks: list[Document], size: int = 200):
    generator = TestsetGenerator.from_langchain(
        llm=init_chat_model(os.environ["JUDGE_MODEL"], temperature=0.3),
        embedding_model=init_embeddings(os.environ["EMBEDDING_MODEL"]),
    )
    testset = generator.generate_with_langchain_docs(chunks, testset_size=size)
    df = testset.to_pandas()          # user_input, reference, reference_contexts, synthesizer_name
    return df[df["user_input"].str.len() > 20]

import asyncio
import os

from openai import AsyncOpenAI
from ragas.llms import llm_factory
from ragas.metrics.collections import Faithfulness

judge = llm_factory(os.environ["JUDGE_MODEL"], provider="openai",
                    client=AsyncOpenAI(base_url=os.environ["JUDGE_BASE_URL"], api_key=os.environ["JUDGE_API_KEY"]))
faithfulness = Faithfulness(llm=judge)


async def score(rows: list[dict]) -> list[float]:
    """rows: {question, answer, contexts} produced by running DocsAssistant on the golden set."""
    results = await asyncio.gather(*(faithfulness.ascore(user_input=r["question"], response=r["answer"],
                                                         retrieved_contexts=r["contexts"]) for r in rows))
    return [r.value for r in results]

import json

import pytest
from deepeval import assert_test
from deepeval.metrics import ContextualPrecisionMetric, FaithfulnessMetric
from deepeval.test_case import LLMTestCase

from docsassistant.eval.runner import run_docsassistant      # returns (answer, retrieved_texts)

GOLDEN = [json.loads(line) for line in open("eval/golden.jsonl")]


@pytest.mark.parametrize("case", GOLDEN, ids=[c["id"] for c in GOLDEN])
def test_golden(case):
    answer, contexts = run_docsassistant(case["question"])
    test_case = LLMTestCase(input=case["question"], actual_output=answer,
                            expected_output=case["reference"], retrieval_context=contexts)
    assert_test(test_case, [FaithfulnessMetric(threshold=0.85), ContextualPrecisionMetric(threshold=0.7)])

from collections import defaultdict
from statistics import mean

TOLERANCE = {"recall_at_5": 0.02, "faithfulness": 0.03, "nugget_recall": 0.03, "citation_validity": 0.01}


def by_slice(rows: list[dict], metric: str) -> dict[str, float]:
    groups: dict[str, list[float]] = defaultdict(list)
    for r in rows:
        groups[r["component"]].append(r[metric])
    return {k: mean(v) for k, v in groups.items()}


def gate(baseline: list[dict], candidate: list[dict]) -> list[str]:
    """Return the list of failures; an empty list means the change may ship."""
    failures = []
    for metric, tol in TOLERANCE.items():
        b, c = mean(r[metric] for r in baseline), mean(r[metric] for r in candidate)
        if c < b - tol:
            failures.append(f"{metric}: {c:.3f} < baseline {b:.3f} - {tol}")
        b_slices, c_slices = by_slice(baseline, metric), by_slice(candidate, metric)
        for name, value in c_slices.items():
            if name in b_slices and value < b_slices[name] - 2 * tol:
                failures.append(f"{metric}[{name}]: {value:.3f} < baseline {b_slices[name]:.3f} - {2 * tol}")
    return failures

package com.example.docsassistant.rag.eval;

import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.List;

import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.document.Document;
import org.springframework.ai.evaluation.EvaluationRequest;
import org.springframework.ai.rag.advisor.RetrievalAugmentationAdvisor;
import org.springframework.beans.factory.annotation.Autowired;

// annotate with @SpringBootTest in the application's test sources
class GoldenSetQualityTest {

    record GoldenCase(String id, String question) {
    }

    @Autowired
    ChatClient docsChatClient;

    @Autowired
    ChatClient.Builder judgeBuilder;

    @Autowired
    List<GoldenCase> goldenSet;          // loaded from eval/golden.jsonl by a test configuration

    @Test
    void passRateDoesNotRegress() {
        var relevancy = RelevancyEvaluator.builder().chatClientBuilder(judgeBuilder).build();
        var factCheck = FactCheckingEvaluator.builder(judgeBuilder).build();
        long passed = goldenSet.stream().filter(c -> {
            ChatResponse response = docsChatClient.prompt().user(c.question()).call().chatResponse();
            List<Document> context = response.getMetadata().get(RetrievalAugmentationAdvisor.DOCUMENT_CONTEXT);
            String answer = response.getResult().getOutput().getText();
            var request = new EvaluationRequest(c.question(), context == null ? List.of() : context, answer);
            return relevancy.evaluate(request).isPass() && factCheck.evaluate(request).isPass();
        }).count();
        double passRate = (double) passed / goldenSet.size();
        assertTrue(passRate >= 0.90, "Golden-set pass rate " + passRate + " is below the 0.90 gate");
    }
}

import hashlib
import math


def sampled(trace_id: str, rate: float, flagged: bool) -> bool:
    """Deterministic in the trace ID, so every service agrees on which requests are judged."""
    if flagged:
        return True
    bucket = int(hashlib.sha256(trace_id.encode()).hexdigest()[:8], 16) / 0xFFFFFFFF
    return bucket < rate


def pass_rate_interval(passed: int, n: int, z: float = 1.96) -> tuple[float, float]:
    p = passed / n
    half = z * math.sqrt(p * (1 - p) / n)
    return max(0.0, p - half), min(1.0, p + half)