build out morpheus-docs MCP stack, mirroring hvm-docs through Phases 1-13

Initial scaffold: the docs-mcp-template clone with all the HVM-validated stack ported across, customized for Morpheus Enterprise (PRODUCT_NAME=morpheus, server name morpheus-docs). Bundles (live-discovered 2026-05-22; 1710 cataloged pages total): * morpheus_user_manual_8_1_0 sd00007510en_us 568 pages (Feb 2026) * morpheus_user_manual_8_1_1 sd00007621en_us 569 pages (Mar 2026) * morpheus_user_manual_8_1_2 sd00007732en_us 569 pages (Apr 2026) * morpheus_release_notes_8_1_0 sd00007496en_us single-doc * morpheus_release_notes_8_1_1 sd00007610en_us single-doc * morpheus_release_notes_8_1_2 sd00007733en_us single-doc * morpheus_quickspecs a50009231enw html-file (live curl_cffi against www.hpe.com; all 12+ Enterprise SKUs captured — S6E64..S6E73AAE for new/renewal/upgrade × 1/3/5-yr terms, plus services SKUs HA124A1#V38/V39 and H46SBA1). No Deployment Guide or Qualification Matrix on HPE Support for Morpheus Enterprise specifically — the only QM (sd00006551en_us) covers HVM clusters managed by Morpheus and lives in hvm-docs. Stack carried forward from hvm-docs: * rag/{index,chunk,embeddings,bm25}.py — including the MAX_CHARS=4000 chunk-cap fix for table-dense content * docs_mcp/{server,usage}.py — 11 MCP tools, BM25-default search, cross-encoder rerank, hybrid behind HYBRID_SEARCH=true, morpheus_api_lessons (renamed from hvm_api_lessons), env-gated submit_doc_bug * docs_mcp/api_lessons.md — Morpheus-specific scaffold covering licensing model, HVM elevation path, REST vs Plugin API, with TODO markers for sections to flesh out from real ops experience * scrape/{runner,quickspecs,changelog,bundles}.py — TOC + single-doc + html-file modes, curl_cffi Chrome120 for www.hpe.com edge bypass * eval/{retrievers,run_eval}.py + queries.jsonl scaffold (4 placeholder queries; populate after first scrape) * scripts/{rerank_server,usage_report,registry_gc}.py * .gitea/workflows/{refresh,image-only}.yml — same Gitea Actions setup zerto-docs uses (push LAN, pull public-URL, GPU Ollama pool) * deploy/docker-compose.yml — morpheus-docs-mcp service definition, shared jina-rerank sidecar, Watchtower-labeled * Dockerfile, requirements.txt, requirements-rerank.txt Verified locally: scrape produced 1599 .md pages (some TOC entries are parent-only and yield no body), 6353 chunks all under the 4 KB cap, MCP server boots and lists 11 tools cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 15:26:24 -04:00
parent 43728320bf
commit fa448f94e1
22 changed files with 2822 additions and 247 deletions
@@ -76,15 +76,87 @@ def main() -> int:
    queries = load_queries(args.queries)
    print(f"loaded {len(queries)} queries")

-    # TODO Phase 7: instantiate the retrievers you implemented in
-    # eval/retrievers.py and run each one against each query.
-    # Aggregate MRR / Recall@K / nDCG@K per retriever. Emit a
-    # markdown table to args.output. Commit the file alongside the
-    # PR that changes retrieval.
-    raise NotImplementedError(
-        "Wire up the retrievers in eval/retrievers.py first, then "
-        "fill in this evaluation loop. See PLAN.md Phase 7."
-    )
+    import os
+    import chromadb
+    from chromadb.config import Settings
+    from rag.embeddings import embedding_function
+    from rag.bm25 import BM25Index
+    from eval.retrievers import DenseRetriever, BM25Retriever, HybridRetriever
+
+    product = os.environ.get("PRODUCT_NAME", "hvm")
+    repo_root = Path(__file__).resolve().parent.parent
+    client = chromadb.PersistentClient(path=str(repo_root / "chroma"),
+                                       settings=Settings(anonymized_telemetry=False))
+    col = client.get_collection(f"{product}_docs", embedding_function=embedding_function())
+    bm = BM25Index(str(repo_root / "bm25" / f"{product}_docs.db"))
+
+    from eval.retrievers import RerankedRetriever
+
+    dense = DenseRetriever(col)
+    bm25 = BM25Retriever(bm)
+    hybrid = HybridRetriever(DenseRetriever(col, pool=100), BM25Retriever(bm, pool=100))
+
+    retrievers = [dense, bm25, hybrid]
+
+    rerank_url = os.environ.get("RERANK_URL", "").rstrip("/")
+    if rerank_url:
+        retrievers += [
+            RerankedRetriever(bm25, col, rerank_url, name_suffix="rerank", pool=50),
+            RerankedRetriever(hybrid, col, rerank_url, name_suffix="rerank", pool=50),
+        ]
+        print(f"reranker enabled: {rerank_url}")
+
+    rows: dict[str, dict[str, float]] = {}
+    per_query: list[dict] = []
+    for r in retrievers:
+        mrr_sum = recall_sum = ndcg_sum = 0.0
+        elapsed_sum = 0.0
+        for q in queries:
+            expected = [(e["bundle_id"], e["page_id"]) for e in q["expected"]]
+            t0 = time.time()
+            retrieved = r.retrieve(q["query"], k=max(args.k, 10))
+            elapsed = time.time() - t0
+            mrr = reciprocal_rank(retrieved, expected)
+            recall = recall_at_k(retrieved, expected, args.k)
+            ndcg = ndcg_at_k(retrieved, expected, args.k)
+            mrr_sum += mrr
+            recall_sum += recall
+            ndcg_sum += ndcg
+            elapsed_sum += elapsed
+            per_query.append({
+                "retriever": r.name, "query": q["query"],
+                "mrr": mrr, "recall@k": recall, "ndcg@k": ndcg,
+                "top1": list(retrieved[0]) if retrieved else None,
+                "elapsed_s": round(elapsed, 3),
+            })
+        n = len(queries)
+        rows[r.name] = {
+            "MRR": mrr_sum / n,
+            f"Recall@{args.k}": recall_sum / n,
+            f"nDCG@{args.k}": ndcg_sum / n,
+            "avg_latency_s": elapsed_sum / n,
+        }
+        print(f"  {r.name}: MRR={rows[r.name]['MRR']:.3f}  "
+              f"Recall@{args.k}={rows[r.name][f'Recall@{args.k}']:.3f}  "
+              f"nDCG@{args.k}={rows[r.name][f'nDCG@{args.k}']:.3f}  "
+              f"avg={rows[r.name]['avg_latency_s']*1000:.0f}ms")
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    md = [f"# Retrieval eval — k={args.k}", "",
+          f"_{len(queries)} hand-curated queries, generated {time.strftime('%Y-%m-%d %H:%M:%S')}_", "",
+          "| Retriever | MRR | Recall@{k} | nDCG@{k} | avg latency |".replace("{k}", str(args.k)),
+          "| --- | ---: | ---: | ---: | ---: |"]
+    for name, m in rows.items():
+        md.append(f"| `{name}` | {m['MRR']:.3f} | {m[f'Recall@{args.k}']:.3f} "
+                  f"| {m[f'nDCG@{args.k}']:.3f} | {m['avg_latency_s']*1000:.0f}ms |")
+    md += ["", "## Per-query results", "",
+           "| Retriever | Query | MRR | top-1 |", "| --- | --- | ---: | --- |"]
+    for r in per_query:
+        top1 = f"`{r['top1'][0]}/{r['top1'][1][:24]}...`" if r["top1"] else "—"
+        md.append(f"| `{r['retriever']}` | {r['query'][:60]} | {r['mrr']:.3f} | {top1} |")
+    args.output.write_text("\n".join(md) + "\n")
+    print(f"wrote {args.output}")
+    return 0


 if __name__ == "__main__":