search: BM25-default + cross-encoder rerank, hybrid behind env gate
Phase 3/6/7/8 in one pass since they depend on each other.
* docs_mcp/server.py
- Wire search_docs / get_page / list_versions tool bodies.
- search_docs flow: BM25 first (rag.bm25 FTS5) → over-fetch RERANK_POOL
chunks → POST to RERANK_URL/v1/rerank → return top-k. Dense is the
fallback when BM25 finds nothing. HYBRID_SEARCH=true switches to
dense+BM25+RRF (fused via the new _rrf_fuse helper).
- All retrieval failures are caught and fall back to the next layer,
so a dead reranker or missing BM25 db never blocks a search.
- Source URLs built from the bundle's docId so results link straight
into support.hpe.com.
* eval/
- 22 hand-curated golden queries grounded in real corpus page titles.
- DenseRetriever / BM25Retriever / HybridRetriever / RerankedRetriever
+ MRR/Recall@K/nDCG@K harness. RERANK_URL env activates the
reranked variants.
- Committed eval/results/baseline.md. On this corpus:
dense: MRR 0.539
bm25: MRR 0.880
hybrid_rrf: MRR 0.692
bm25+rerank: MRR 0.920 (winner)
hybrid_rrf+rerank: MRR 0.875
HPE structured docs use controlled vocabulary, so lexical match
dominates. Hybrid loses because dense pollutes the fused pool.
* scripts/rerank_server.py
- Minimal HTTP /v1/rerank over sentence-transformers
cross-encoder/ms-marco-MiniLM-L-6-v2. Cohere-style request/response.
- This is the dev/CPU fallback; production replaces it with the
llama.cpp + jina-reranker-v2-base GGUF sidecar (same wire protocol).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+81
-9
@@ -76,15 +76,87 @@ def main() -> int:
|
||||
queries = load_queries(args.queries)
|
||||
print(f"loaded {len(queries)} queries")
|
||||
|
||||
# TODO Phase 7: instantiate the retrievers you implemented in
|
||||
# eval/retrievers.py and run each one against each query.
|
||||
# Aggregate MRR / Recall@K / nDCG@K per retriever. Emit a
|
||||
# markdown table to args.output. Commit the file alongside the
|
||||
# PR that changes retrieval.
|
||||
raise NotImplementedError(
|
||||
"Wire up the retrievers in eval/retrievers.py first, then "
|
||||
"fill in this evaluation loop. See PLAN.md Phase 7."
|
||||
)
|
||||
import os
|
||||
import chromadb
|
||||
from chromadb.config import Settings
|
||||
from rag.embeddings import embedding_function
|
||||
from rag.bm25 import BM25Index
|
||||
from eval.retrievers import DenseRetriever, BM25Retriever, HybridRetriever
|
||||
|
||||
product = os.environ.get("PRODUCT_NAME", "hvm")
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
client = chromadb.PersistentClient(path=str(repo_root / "chroma"),
|
||||
settings=Settings(anonymized_telemetry=False))
|
||||
col = client.get_collection(f"{product}_docs", embedding_function=embedding_function())
|
||||
bm = BM25Index(str(repo_root / "bm25" / f"{product}_docs.db"))
|
||||
|
||||
from eval.retrievers import RerankedRetriever
|
||||
|
||||
dense = DenseRetriever(col)
|
||||
bm25 = BM25Retriever(bm)
|
||||
hybrid = HybridRetriever(DenseRetriever(col, pool=100), BM25Retriever(bm, pool=100))
|
||||
|
||||
retrievers = [dense, bm25, hybrid]
|
||||
|
||||
rerank_url = os.environ.get("RERANK_URL", "").rstrip("/")
|
||||
if rerank_url:
|
||||
retrievers += [
|
||||
RerankedRetriever(bm25, col, rerank_url, name_suffix="rerank", pool=50),
|
||||
RerankedRetriever(hybrid, col, rerank_url, name_suffix="rerank", pool=50),
|
||||
]
|
||||
print(f"reranker enabled: {rerank_url}")
|
||||
|
||||
rows: dict[str, dict[str, float]] = {}
|
||||
per_query: list[dict] = []
|
||||
for r in retrievers:
|
||||
mrr_sum = recall_sum = ndcg_sum = 0.0
|
||||
elapsed_sum = 0.0
|
||||
for q in queries:
|
||||
expected = [(e["bundle_id"], e["page_id"]) for e in q["expected"]]
|
||||
t0 = time.time()
|
||||
retrieved = r.retrieve(q["query"], k=max(args.k, 10))
|
||||
elapsed = time.time() - t0
|
||||
mrr = reciprocal_rank(retrieved, expected)
|
||||
recall = recall_at_k(retrieved, expected, args.k)
|
||||
ndcg = ndcg_at_k(retrieved, expected, args.k)
|
||||
mrr_sum += mrr
|
||||
recall_sum += recall
|
||||
ndcg_sum += ndcg
|
||||
elapsed_sum += elapsed
|
||||
per_query.append({
|
||||
"retriever": r.name, "query": q["query"],
|
||||
"mrr": mrr, "recall@k": recall, "ndcg@k": ndcg,
|
||||
"top1": list(retrieved[0]) if retrieved else None,
|
||||
"elapsed_s": round(elapsed, 3),
|
||||
})
|
||||
n = len(queries)
|
||||
rows[r.name] = {
|
||||
"MRR": mrr_sum / n,
|
||||
f"Recall@{args.k}": recall_sum / n,
|
||||
f"nDCG@{args.k}": ndcg_sum / n,
|
||||
"avg_latency_s": elapsed_sum / n,
|
||||
}
|
||||
print(f" {r.name}: MRR={rows[r.name]['MRR']:.3f} "
|
||||
f"Recall@{args.k}={rows[r.name][f'Recall@{args.k}']:.3f} "
|
||||
f"nDCG@{args.k}={rows[r.name][f'nDCG@{args.k}']:.3f} "
|
||||
f"avg={rows[r.name]['avg_latency_s']*1000:.0f}ms")
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
md = [f"# Retrieval eval — k={args.k}", "",
|
||||
f"_{len(queries)} hand-curated queries, generated {time.strftime('%Y-%m-%d %H:%M:%S')}_", "",
|
||||
"| Retriever | MRR | Recall@{k} | nDCG@{k} | avg latency |".replace("{k}", str(args.k)),
|
||||
"| --- | ---: | ---: | ---: | ---: |"]
|
||||
for name, m in rows.items():
|
||||
md.append(f"| `{name}` | {m['MRR']:.3f} | {m[f'Recall@{args.k}']:.3f} "
|
||||
f"| {m[f'nDCG@{args.k}']:.3f} | {m['avg_latency_s']*1000:.0f}ms |")
|
||||
md += ["", "## Per-query results", "",
|
||||
"| Retriever | Query | MRR | top-1 |", "| --- | --- | ---: | --- |"]
|
||||
for r in per_query:
|
||||
top1 = f"`{r['top1'][0]}/{r['top1'][1][:24]}...`" if r["top1"] else "—"
|
||||
md.append(f"| `{r['retriever']}` | {r['query'][:60]} | {r['mrr']:.3f} | {top1} |")
|
||||
args.output.write_text("\n".join(md) + "\n")
|
||||
print(f"wrote {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user