Port docs-mcp-template upgrades (eval, citations, chunking, prefixes) (#13)

Co-authored-by: claude <[email protected]>

eval/run_eval.py

@@ -34,6 +34,12 @@ def load_queries(path: Path) -> list[dict]:
         return [json.loads(line) for line in fh if line.strip()]
 
 
+def p_at_1(retrieved: list[tuple[str, str]], expected: list[tuple[str, str]]) -> float:
+    if not retrieved or not expected:
+        return 0.0
+    return 1.0 if retrieved[0] in set(expected) else 0.0
+
+
 def reciprocal_rank(retrieved: list[tuple[str, str]], expected: list[tuple[str, str]]) -> float:
     expected_set = set(expected)
     for i, page in enumerate(retrieved, start=1):
@@ -65,8 +71,12 @@ def main() -> int:
     p = argparse.ArgumentParser()
     p.add_argument("--queries", type=Path, default=Path("eval/queries.jsonl"))
     p.add_argument("--k", type=int, default=5)
+    p.add_argument("--ks", default="1,5,10,20", help="comma-separated k-curve")
     p.add_argument("--output", type=Path, default=Path("eval/results/baseline.md"))
+    p.add_argument("--compare", type=Path, default=None)
     args = p.parse_args()
+    ks = sorted({int(x) for x in args.ks.split(",") if x.strip()}) or [args.k]
+    max_k = max(ks + [args.k])
 
     if not args.queries.exists():
         print(f"queries file not found: {args.queries}")
@@ -83,7 +93,7 @@ def main() -> int:
     from rag.bm25 import BM25Index
     from eval.retrievers import DenseRetriever, BM25Retriever, HybridRetriever
 
-    product = os.environ.get("PRODUCT_NAME", "hvm")
+    product = os.environ.get("PRODUCT_NAME", "morpheus")
     repo_root = Path(__file__).resolve().parent.parent
     client = chromadb.PersistentClient(path=str(repo_root / "chroma"),
                                        settings=Settings(anonymized_telemetry=False))
@@ -109,34 +119,39 @@ def main() -> int:
     rows: dict[str, dict[str, float]] = {}
     per_query: list[dict] = []
     for r in retrievers:
-        mrr_sum = recall_sum = ndcg_sum = 0.0
+        mrr_sum = recall_sum = ndcg_sum = p1_sum = 0.0
         elapsed_sum = 0.0
         for q in queries:
             expected = [(e["bundle_id"], e["page_id"]) for e in q["expected"]]
             t0 = time.time()
-            retrieved = r.retrieve(q["query"], k=max(args.k, 10))
+            retrieved = r.retrieve(q["query"], k=max(max_k, 10))
             elapsed = time.time() - t0
             mrr = reciprocal_rank(retrieved, expected)
+            p1 = p_at_1(retrieved, expected)
             recall = recall_at_k(retrieved, expected, args.k)
             ndcg = ndcg_at_k(retrieved, expected, args.k)
             mrr_sum += mrr
+            p1_sum += p1
             recall_sum += recall
             ndcg_sum += ndcg
             elapsed_sum += elapsed
             per_query.append({
                 "retriever": r.name, "query": q["query"],
-                "mrr": mrr, "recall@k": recall, "ndcg@k": ndcg,
+                "mrr": mrr, "p_at_1": int(p1), "recall@k": recall, "ndcg@k": ndcg,
                 "top1": list(retrieved[0]) if retrieved else None,
+                "ranked": [list(p) for p in retrieved],
                 "elapsed_s": round(elapsed, 3),
             })
         n = len(queries)
         rows[r.name] = {
+            "P@1": p1_sum / n,
             "MRR": mrr_sum / n,
             f"Recall@{args.k}": recall_sum / n,
             f"nDCG@{args.k}": ndcg_sum / n,
             "avg_latency_s": elapsed_sum / n,
         }
-        print(f"  {r.name}: MRR={rows[r.name]['MRR']:.3f}  "
+        print(f"  {r.name}: P@1={rows[r.name]['P@1']:.3f}  "
+              f"MRR={rows[r.name]['MRR']:.3f}  "
               f"Recall@{args.k}={rows[r.name][f'Recall@{args.k}']:.3f}  "
               f"nDCG@{args.k}={rows[r.name][f'nDCG@{args.k}']:.3f}  "
               f"avg={rows[r.name]['avg_latency_s']*1000:.0f}ms")
@@ -144,10 +159,10 @@ def main() -> int:
     args.output.parent.mkdir(parents=True, exist_ok=True)
     md = [f"# Retrieval eval — k={args.k}", "",
           f"_{len(queries)} hand-curated queries, generated {time.strftime('%Y-%m-%d %H:%M:%S')}_", "",
-          "| Retriever | MRR | Recall@{k} | nDCG@{k} | avg latency |".replace("{k}", str(args.k)),
-          "| --- | ---: | ---: | ---: | ---: |"]
+          "| Retriever | P@1 | MRR | Recall@{k} | nDCG@{k} | avg latency |".replace("{k}", str(args.k)),
+          "| --- | ---: | ---: | ---: | ---: | ---: |"]
     for name, m in rows.items():
-        md.append(f"| `{name}` | {m['MRR']:.3f} | {m[f'Recall@{args.k}']:.3f} "
+        md.append(f"| `{name}` | {m['P@1']:.3f} | {m['MRR']:.3f} | {m[f'Recall@{args.k}']:.3f} "
                   f"| {m[f'nDCG@{args.k}']:.3f} | {m['avg_latency_s']*1000:.0f}ms |")
     md += ["", "## Per-query results", "",
            "| Retriever | Query | MRR | top-1 |", "| --- | --- | ---: | --- |"]
@@ -155,7 +170,22 @@ def main() -> int:
         top1 = f"`{r['top1'][0]}/{r['top1'][1][:24]}...`" if r["top1"] else "—"
         md.append(f"| `{r['retriever']}` | {r['query'][:60]} | {r['mrr']:.3f} | {top1} |")
     args.output.write_text("\n".join(md) + "\n")
+    sidecar = args.output.with_suffix(".jsonl")
+    with open(sidecar, "w") as fh:
+        for r in per_query:
+            fh.write(json.dumps({
+                "query": r["query"],
+                "retriever": r["retriever"],
+                "ranked": r.get("ranked") or [],
+                "rr": r["mrr"],
+                "p_at_1": r["p_at_1"],
+            }) + "\n")
     print(f"wrote {args.output}")
+    print(f"wrote {sidecar}")
+    if args.compare:
+        from eval.pvalue import compare, load_sidecar, render
+        print()
+        print(render(compare(load_sidecar(sidecar), load_sidecar(args.compare))), end="")
     return 0