Choosing a Model

import anthropic

client = anthropic.Anthropic()  # reads ANTHROPIC_API_KEY from the environment

EXTRACT_PAGE_METADATA = {
    "name": "extract_page_metadata",
    "description": "Extract structured metadata from a DocsAssistant page.",
    "strict": True,
    "input_schema": {
        "type": "object",
        "properties": {
            "title": {"type": "string"},
            "component": {"type": "string"},
            "topics": {"type": "array", "items": {"type": "string"}},
        },
        "required": ["title", "component", "topics"],
        "additionalProperties": False,
    },
}

# Five DocsAssistant pages, each with the metadata a correct extraction should return.
EVAL_CASES = [
    {
        "page_text": "== Configuring HNSW parameters\nThis page (component: pgvector-guide) covers "
                     "ef_search, ef_construction and m for pgvector HNSW indexes.",
        "expected": {
            "title": "Configuring HNSW parameters",
            "component": "pgvector-guide",
            "topics": ["ef_search", "ef_construction", "m"],
        },
    },
    {
        "page_text": "== Prompt caching with cache_control\nThis page (component: llm-foundations) covers "
                     "ephemeral cache breakpoints and the 5-minute and 1-hour TTLs.",
        "expected": {
            "title": "Prompt caching with cache_control",
            "component": "llm-foundations",
            "topics": ["ephemeral cache breakpoints", "5-minute TTL", "1-hour TTL"],
        },
    },
    {
        "page_text": "== Retrieval Augmented Generation basics\nThis page (component: vector-rag) covers "
                     "chunking, retrieval and prompt augmentation.",
        "expected": {
            "title": "Retrieval Augmented Generation basics",
            "component": "vector-rag",
            "topics": ["chunking", "retrieval", "prompt augmentation"],
        },
    },
    {
        "page_text": "== Spring AI ChatClient basics\nThis page (component: spring-ai) covers prompt(), "
                     "call() and entity().",
        "expected": {
            "title": "Spring AI ChatClient basics",
            "component": "spring-ai",
            "topics": ["prompt()", "call()", "entity()"],
        },
    },
    {
        "page_text": "== Sampling with temperature and top-p\nThis page (component: llm-foundations) covers "
                     "temperature, top-p and top-k.",
        "expected": {
            "title": "Sampling with temperature and top-p",
            "component": "llm-foundations",
            "topics": ["temperature", "top-p", "top-k"],
        },
    },
]


def extract(model: str, page_text: str) -> dict:
    response = client.messages.create(
        model=model,
        max_tokens=1024,
        tools=[EXTRACT_PAGE_METADATA],
        tool_choice={"type": "tool", "name": "extract_page_metadata"},
        messages=[{"role": "user", "content": f"Extract metadata from this page:\n\n{page_text}"}],
    )
    tool_use = next(block for block in response.content if block.type == "tool_use")
    return tool_use.input


def exact_match_score(actual: dict, expected: dict) -> float:
    """Fraction of the expected fields the actual answer matched exactly."""
    fields = expected.keys()
    matches = sum(1 for field in fields if actual.get(field) == expected[field])
    return matches / len(fields)


def run_eval(models: list[str]) -> dict[str, float]:
    scores = {model: 0.0 for model in models}
    for case in EVAL_CASES:
        for model in models:
            actual = extract(model, case["page_text"])
            scores[model] += exact_match_score(actual, case["expected"])
    return {model: total / len(EVAL_CASES) for model, total in scores.items()}


average_scores = run_eval(["claude-haiku-4-5", "claude-sonnet-5"])
print(average_scores)  # e.g. {'claude-haiku-4-5': 0.87, 'claude-sonnet-5': 1.0}

import json
import anthropic

client = anthropic.Anthropic()  # reads ANTHROPIC_API_KEY from the environment

FAST_MODEL = "claude-haiku-4-5"
STRONG_MODEL = "claude-sonnet-5"
CONFIDENCE_THRESHOLD = 0.7

PROMPT_TEMPLATE = (
    "Extract title, component and topics as JSON from this DocsAssistant page. Include a "
    '"confidence" field between 0 and 1 for how sure you are of the extraction. Return JSON '
    "only, no other text.\n\n{page_text}"
)


def parse_json_reply(text: str) -> dict | None:
    """Parse a JSON reply, tolerating a Markdown code fence (```json ... ```) around it."""
    text = text.strip()
    if text.startswith("```"):
        text = text.split("\n", 1)[1] if "\n" in text else ""
        text = text.rsplit("```", 1)[0]
    try:
        parsed = json.loads(text)
    except json.JSONDecodeError:
        return None
    return parsed if isinstance(parsed, dict) else None


def call_model(model: str, page_text: str) -> dict | None:
    response = client.messages.create(
        model=model,
        max_tokens=1024,
        messages=[{"role": "user", "content": PROMPT_TEMPLATE.format(page_text=page_text)}],
    )
    return parse_json_reply(response.content[0].text)  # None = failed attempt, see below


def extract_with_cascade(page_text: str) -> dict:
    result = call_model(FAST_MODEL, page_text)
    needs_escalation = result is None or result.get("confidence", 0) < CONFIDENCE_THRESHOLD
    if needs_escalation:
        result = call_model(STRONG_MODEL, page_text)
    if result is None:
        # Even the strong model returned unparseable text: fail loudly instead of handing None
        # to the caller (or fall back to the strict tool call used by the harness above).
        raise ValueError("Both models failed to return valid JSON for this page.")
    return result