seed-mcp/eval/run_eval.py

"""Run all retrievers against eval/queries.jsonl, emit a markdown report.

Metrics computed per retriever:

  MRR        — mean reciprocal rank of the FIRST expected page in the
               ranked result list (0 if not in top-k).
  Recall@K   — fraction of expected pages that appear in top-K.
  nDCG@K     — discounted gain weighted by rank position.

The "right" number depends on what you're measuring. MRR tracks "the
first-line answer is correct"; Recall@K tracks "everything relevant
is there to draw from"; nDCG@K is a smoother combination of both.
For docs-RAG, MRR is usually the headline metric.

Usage:

    python -m eval.run_eval \\
        --queries eval/queries.jsonl \\
        --k 5 \\
        --output eval/results/baseline.md
"""
from __future__ import annotations

import argparse
import json
import math
import time
from pathlib import Path
from typing import Iterable


def load_queries(path: Path) -> list[dict]:
    with open(path) as fh:
        return [json.loads(line) for line in fh if line.strip()]


def reciprocal_rank(retrieved: list[tuple[str, str]], expected: list[tuple[str, str]]) -> float:
    expected_set = set(expected)
    for i, page in enumerate(retrieved, start=1):
        if page in expected_set:
            return 1.0 / i
    return 0.0


def recall_at_k(retrieved: list[tuple[str, str]], expected: list[tuple[str, str]], k: int) -> float:
    if not expected:
        return 0.0
    retrieved_set = set(retrieved[:k])
    hits = sum(1 for e in expected if e in retrieved_set)
    return hits / len(expected)


def ndcg_at_k(retrieved: list[tuple[str, str]], expected: list[tuple[str, str]], k: int) -> float:
    expected_set = set(expected)
    dcg = 0.0
    for i, page in enumerate(retrieved[:k], start=1):
        if page in expected_set:
            dcg += 1.0 / math.log2(i + 1)
    # Ideal DCG: every expected page in the top positions.
    idcg = sum(1.0 / math.log2(i + 1) for i in range(1, min(len(expected), k) + 1))
    return dcg / idcg if idcg else 0.0


def main() -> int:
    p = argparse.ArgumentParser()
    p.add_argument("--queries", type=Path, default=Path("eval/queries.jsonl"))
    p.add_argument("--k", type=int, default=5)
    p.add_argument("--output", type=Path, default=Path("eval/results/baseline.md"))
    args = p.parse_args()

    if not args.queries.exists():
        print(f"queries file not found: {args.queries}")
        print("hint: copy eval/queries.jsonl.example and edit")
        return 1

    queries = load_queries(args.queries)
    print(f"loaded {len(queries)} queries")

    # TODO Phase 7: instantiate the retrievers you implemented in
    # eval/retrievers.py and run each one against each query.
    # Aggregate MRR / Recall@K / nDCG@K per retriever. Emit a
    # markdown table to args.output. Commit the file alongside the
    # PR that changes retrieval.
    raise NotImplementedError(
        "Wire up the retrievers in eval/retrievers.py first, then "
        "fill in this evaluation loop. See PLAN.md Phase 7."
    )


if __name__ == "__main__":
    raise SystemExit(main())