Port docs-mcp-template upgrades (eval, citations, prefixes) (#26)

Co-authored-by: claude <[email protected]>

eval/pvalue.py

@@ -0,0 +1,156 @@
+"""Paired permutation test between two eval JSONL sidecars.
+
+Compares per-query scores from two `eval.run_eval` sidecar files so
+"P@1 went 0.88 → 0.91 on 25 queries" is not treated as a win.
+
+    python -m eval.pvalue \\
+        --a eval/results/baseline.jsonl \\
+        --b eval/results/new.jsonl \\
+        --metric rr
+
+Exit 0 even when the difference is not significant — this is a report,
+not a gate. No third-party deps; `random.Random(seed)` is enough.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+from pathlib import Path
+
+
+def load_sidecar(path: Path) -> list[dict]:
+    rows: list[dict] = []
+    with open(path) as fh:
+        for line in fh:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def paired_permutation(
+    a: list[float],
+    b: list[float],
+    n_resamples: int = 10000,
+    seed: int = 0,
+) -> dict:
+    """Two-sided paired permutation test on per-query scores.
+
+    Null: each pair is exchangeable (randomly flipping the sign of
+    A_i - B_i). p_value is the fraction of permutations whose
+    |mean diff| is at least as large as the observed |mean(A-B)|.
+    """
+    if len(a) != len(b):
+        raise ValueError(f"paired lengths differ: {len(a)} vs {len(b)}")
+    if not a:
+        raise ValueError("no paired queries to compare")
+    diffs = [x - y for x, y in zip(a, b)]
+    n = len(diffs)
+    observed = sum(diffs) / n
+    abs_obs = abs(observed)
+    rng = random.Random(seed)
+    extreme = 0
+    for _ in range(n_resamples):
+        total = 0.0
+        for d in diffs:
+            total += d if rng.random() < 0.5 else -d
+        if abs(total / n) >= abs_obs - 1e-15:
+            extreme += 1
+    p_value = extreme / n_resamples
+    return {
+        "A_mean": sum(a) / n,
+        "B_mean": sum(b) / n,
+        "Diff(A-B)": observed,
+        "p_value": p_value,
+        "significant": p_value < 0.05,
+        "n": n,
+        "n_resamples": n_resamples,
+    }
+
+
+def _index(rows: list[dict], metric: str) -> dict[tuple[str, str], float]:
+    """Map (retriever, query) -> score."""
+    out: dict[tuple[str, str], float] = {}
+    for row in rows:
+        retriever = str(row.get("retriever") or "")
+        query = str(row.get("query") or "")
+        if metric == "p_at_1":
+            score = float(row.get("p_at_1") or 0)
+        else:
+            score = float(row.get("rr") or 0)
+        out[(retriever, query)] = score
+    return out
+
+
+def compare(
+    rows_a: list[dict],
+    rows_b: list[dict],
+    metric: str = "rr",
+    retriever: str | None = None,
+    n_resamples: int = 10000,
+    seed: int = 0,
+) -> list[dict]:
+    """Join on (retriever, query). One result dict per shared retriever."""
+    ia, ib = _index(rows_a, metric), _index(rows_b, metric)
+    retrievers = sorted({r for r, _ in ia} & {r for r, _ in ib})
+    if retriever:
+        retrievers = [r for r in retrievers if r == retriever]
+        if not retrievers:
+            raise ValueError(f"retriever {retriever!r} not in both sidecars")
+    reports = []
+    for name in retrievers:
+        queries = sorted({q for r, q in ia if r == name} & {q for r, q in ib if r == name})
+        if not queries:
+            continue
+        a_scores = [ia[(name, q)] for q in queries]
+        b_scores = [ib[(name, q)] for q in queries]
+        report = paired_permutation(a_scores, b_scores, n_resamples=n_resamples, seed=seed)
+        report["retriever"] = name
+        report["metric"] = metric
+        reports.append(report)
+    if not reports:
+        raise ValueError("no overlapping (retriever, query) pairs")
+    return reports
+
+
+def render(reports: list[dict]) -> str:
+    lines = ["# Permutation test", ""]
+    for r in reports:
+        sig = "yes" if r["significant"] else "no"
+        lines += [
+            f"## `{r['retriever']}` ({r['metric']}, n={r['n']})",
+            "",
+            f"- A_mean: `{r['A_mean']:.4f}`",
+            f"- B_mean: `{r['B_mean']:.4f}`",
+            f"- Diff(A-B): `{r['Diff(A-B)']:.4f}`",
+            f"- p_value: `{r['p_value']:.4f}` ({r['n_resamples']} resamples)",
+            f"- significant (p < 0.05): **{sig}**",
+            "",
+        ]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description="Paired permutation test on two eval JSONL sidecars.")
+    p.add_argument("--a", type=Path, required=True, help="sidecar JSONL (system A)")
+    p.add_argument("--b", type=Path, required=True, help="sidecar JSONL (system B)")
+    p.add_argument("--metric", choices=("rr", "p_at_1"), default="rr")
+    p.add_argument("--retriever", default=None, help="restrict to one retriever name")
+    p.add_argument("--n-resamples", type=int, default=10000)
+    p.add_argument("--seed", type=int, default=0)
+    args = p.parse_args()
+    reports = compare(
+        load_sidecar(args.a),
+        load_sidecar(args.b),
+        metric=args.metric,
+        retriever=args.retriever,
+        n_resamples=args.n_resamples,
+        seed=args.seed,
+    )
+    print(render(reports), end="")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

eval/results/pre-template.md

@@ -0,0 +1,41 @@
+# seed-mcp retrieval eval — k=5
+
+_21 golden queries × 4 retrievers_
+
+## Summary
+
+| Retriever | Passed | Recall | P@1 | MRR | Avg ms |
+|---|---|---|---|---|---|
+| **hybrid+rerank** | 21/21 | 100.00% | 90.48% | 0.905 | 2064 |
+| **bm25** | 20/21 | 95.24% | 80.95% | 0.833 | 5 |
+| **hybrid** | 15/21 | 71.43% | 61.90% | 0.619 | 73 |
+| **dense** | 14/21 | 66.67% | 38.10% | 0.440 | 79 |
+
+**Recall** = % of queries where ≥1 top-k chunk satisfied the spec. **P@1** = % where the very first result satisfied it. **MRR** = mean of `1 / rank-of-first-satisfying-result` (0 if missed).
+
+## Per-query results
+
+| Query | bm25 | dense | hybrid | hybrid+rerank |
+|---|---|---|---|---|
+| `DKC62-08RIB ratings` | ✅ #1 | ❌ | ❌ | ✅ #1 |
+| `AG29XF4 disease ratings` | ✅ #1 | ❌ | ❌ | ✅ #1 |
+| `WB6430 westbred wheat` | ✅ #1 | ❌ | ❌ | ✅ #1 |
+| `E085Z5 corn` | ✅ #1 | ❌ | ❌ | ✅ #1 |
+| `AP Iliad wheat performance` | ✅ #1 | ❌ | ❌ | ✅ #1 |
+| `drought tolerant corn for sandy soil short season Iowa` | ✅ #2 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `soybean cyst nematode SCN resistant variety` | ✅ #1 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `Phytophthora resistance Rps3a soybean` | ✅ #1 | ✅ #2 | ✅ #1 | ✅ #1 |
+| `XtendFlex soybean Northern Plains` | ❌ | ✅ #1 | ✅ #1 | ✅ #1 |
+| `Hard Red Spring wheat stripe rust resistance` | ✅ #1 | ✅ #3 | ✅ #1 | ✅ #1 |
+| `Soft White Winter wheat Pacific Northwest` | ✅ #1 | ✅ #5 | ✅ #1 | ✅ #1 |
+| `Goss's Wilt resistance corn` | ✅ #1 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `best corn 2024 Iowa` | ✅ #1 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `Indiana corn yield comparison 2024` | ✅ #1 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `AP Iliad Idaho wheat trial` | ✅ #1 | ✅ #5 | ✅ #1 | ✅ #1 |
+| `DKC65-95 corn yield in trials` | ✅ #1 | ❌ | ✅ #1 | ✅ #1 |
+| `NK1701 corn trials head to head` | ✅ #1 | ❌ | ❌ | ✅ #1 |
+| `silage corn high milk per acre dairy` | ✅ #1 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `soybean 2025 Minnesota top performers` | ✅ #1 | ✅ #1 | ✅ #1 | ✅ #1 |
+| `Pioneer P1142 hybrid recommendation` | ✅ | ✅ | ✅ | ✅ |
+| `DKC65-20 yield Alabama trial` | ✅ | ✅ | ✅ | ✅ |
+

eval/retrievers.py

@@ -71,8 +71,10 @@ class DenseRetriever:
     def retrieve(self, query: str, k: int, filters: dict | None) -> list[str]:
         where = _build_where(filters)
         try:
+            from rag.embeddings import EMBED_QUERY_PREFIX, embed_texts
+            qvec = embed_texts([query], prefix=EMBED_QUERY_PREFIX)[0]
             r = self.col.query(
-                query_texts=[query], n_results=max(k, self.pool), where=where,
+                query_embeddings=[qvec], n_results=max(k, self.pool), where=where,
             )
         except Exception:
             return []
@@ -105,7 +107,9 @@ class HybridRetriever:
     def retrieve(self, query: str, k: int, filters: dict | None) -> list[str]:
         where = _build_where(filters)
         try:
-            d = self.col.query(query_texts=[query], n_results=self.pool, where=where)
+            from rag.embeddings import EMBED_QUERY_PREFIX, embed_texts
+            qvec = embed_texts([query], prefix=EMBED_QUERY_PREFIX)[0]
+            d = self.col.query(query_embeddings=[qvec], n_results=self.pool, where=where)
             dense_ids = (d.get("ids") or [[]])[0]
         except Exception:
             dense_ids = []
@@ -138,8 +142,10 @@ class HybridRerankRetriever:
     def retrieve(self, query: str, k: int, filters: dict | None) -> list[str]:
         where = _build_where(filters)
         try:
+            from rag.embeddings import EMBED_QUERY_PREFIX, embed_texts
+            qvec = embed_texts([query], prefix=EMBED_QUERY_PREFIX)[0]
             d = self.col.query(
-                query_texts=[query], n_results=self.pool, where=where,
+                query_embeddings=[qvec], n_results=self.pool, where=where,
                 include=["documents"],
             )
             dense_ids = (d.get("ids") or [[]])[0]

eval/run_eval.py

@@ -265,10 +265,15 @@ def main() -> int:
     p = argparse.ArgumentParser()
     p.add_argument("--queries", type=Path, default=Path("eval/queries.jsonl"))
     p.add_argument("--k", type=int, default=5)
+    p.add_argument("--ks", default="1,5,10,20", help="comma-separated k-curve")
     p.add_argument("--output", type=Path, default=Path("eval/results/baseline.md"))
+    p.add_argument("--compare", type=Path, default=None)
+    p.add_argument("--trace", action="store_true")
     p.add_argument("--rerank-url", default=os.environ.get("RERANK_URL", ""))
     p.add_argument("--product-name", default=os.environ.get("PRODUCT_NAME", "crop_seed"))
     args = p.parse_args()
+    ks = sorted({int(x) for x in args.ks.split(",") if x.strip()}) or [args.k]
+    max_k = max(ks + [args.k])
 
     if not args.queries.exists():
         print(f"queries file not found: {args.queries}")
@@ -300,14 +305,40 @@ def main() -> int:
     for r in retrievers:
         print(f"running {r.name}...")
         for q in queries:
-            res = _evaluate_one(r, q, args.k, col)
+            res = _evaluate_one(r, q, max_k, col)
             all_results.append(res)
 
     summary = _aggregate(all_results)
     md = _emit_markdown(queries, all_results, summary, args.k)
+    # k-curve from rank_first_match at max_k
+    md += "\n## k-curve (P@1 / recall from rank_first_match)\n\n"
+    md += "| Retriever | " + " | ".join(f"P@1@k={k}" for k in ks) + " |\n"
+    md += "|" + "---|" * (len(ks) + 1) + "\n"
+    by_r: dict[str, list[dict]] = {}
+    for row in all_results:
+        by_r.setdefault(row["retriever"], []).append(row)
+    for name, rows in by_r.items():
+        cells = []
+        for kk in ks:
+            hits = sum(1 for r in rows if r.get("rank_first_match") and r["rank_first_match"] <= kk)
+            cells.append(f"{hits / len(rows):.3f}" if rows else "0")
+        md += f"| `{name}` | " + " | ".join(cells) + " |\n"
     args.output.parent.mkdir(parents=True, exist_ok=True)
     args.output.write_text(md, encoding="utf-8")
+    sidecar = args.output.with_suffix(".jsonl")
+    with open(sidecar, "w") as fh:
+        for r in all_results:
+            rank = r.get("rank_first_match")
+            fh.write(json.dumps({
+                "query": r["query"],
+                "retriever": r["retriever"],
+                "passed": bool(r.get("passed")),
+                "p_at_1": 1 if rank == 1 else 0,
+                "rr": (1.0 / rank) if rank else 0.0,
+                "rank_first_match": rank,
+            }) + "\n")
     print(f"\nreport: {args.output}")
+    print(f"sidecar: {sidecar}")
     print()
     # Print summary to stdout too
     for line in md.split("\n"):
@@ -315,6 +346,15 @@ def main() -> int:
             print(line)
         if line.startswith("## Per-query"):
             break
+    if args.compare:
+        from eval.pvalue import compare, load_sidecar, render
+        print()
+        print(render(compare(load_sidecar(sidecar), load_sidecar(args.compare))), end="")
+    if args.trace:
+        from eval.trace import render_misses
+        misses_path = args.output.with_name("misses.md")
+        misses_path.write_text(render_misses(all_results))
+        print(f"misses: {misses_path}")
     return 0
 
 

eval/test_pvalue.py

@@ -0,0 +1,25 @@
+"""Stdlib tests for the permutation test. No Chroma."""
+from __future__ import annotations
+
+import unittest
+
+from eval.pvalue import paired_permutation
+
+
+class PermutationTests(unittest.TestCase):
+    def test_identical_is_not_significant(self) -> None:
+        a = [1.0, 0.5, 0.0, 1.0]
+        r = paired_permutation(a, list(a), n_resamples=200, seed=0)
+        self.assertEqual(r["Diff(A-B)"], 0.0)
+        self.assertFalse(r["significant"])
+
+    def test_large_shift_is_significant(self) -> None:
+        a = [1.0] * 20
+        b = [0.0] * 20
+        r = paired_permutation(a, b, n_resamples=500, seed=0)
+        self.assertTrue(r["significant"])
+        self.assertLess(r["p_value"], 0.05)
+
+
+if __name__ == "__main__":
+    unittest.main()

eval/trace.py

@@ -0,0 +1,50 @@
+"""Miss dump from an eval sidecar or in-memory rows.
+
+    python -m eval.trace --sidecar eval/results/baseline.jsonl
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def render_misses(rows: list[dict]) -> str:
+    misses = [r for r in rows if not r.get("passed") and r.get("p_at_1", 1) == 0]
+    # passed=False is the seed schema; p_at_1==0 covers pvalue sidecars
+    if not misses:
+        misses = [r for r in rows if not r.get("passed")]
+    if not misses:
+        return "# Eval misses\n\n_(none)_\n"
+    lines = [f"# Eval misses ({len(misses)})", ""]
+    for row in misses:
+        lines += [
+            f"## {row.get('query', '')}",
+            "",
+            f"- retriever: `{row.get('retriever')}`",
+            f"- rank_first_match: `{row.get('rank_first_match')}`",
+            f"- kind: `{row.get('kind', '')}`",
+            "",
+        ]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--sidecar", type=Path, default=Path("eval/results/baseline.jsonl"))
+    p.add_argument("--misses-out", type=Path, default=Path("eval/results/misses.md"))
+    args = p.parse_args()
+    if not args.sidecar.exists():
+        args.misses_out.parent.mkdir(parents=True, exist_ok=True)
+        args.misses_out.write_text("# Eval misses\n\nno sidecar\n")
+        print(f"no sidecar ({args.sidecar}); wrote empty misses")
+        return 0
+    rows = [json.loads(line) for line in args.sidecar.read_text().splitlines() if line.strip()]
+    args.misses_out.parent.mkdir(parents=True, exist_ok=True)
+    args.misses_out.write_text(render_misses(rows))
+    print(f"wrote {args.misses_out} ({len(rows)} rows)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())