Port docs-mcp-template upgrades (eval, citations, prefixes) (#26)
Image rebuild (skip scrape) / build (push) Successful in 3m58s
Image rebuild (skip scrape) / build (push) Successful in 3m58s
Co-authored-by: claude <[email protected]>
This commit was merged in pull request #26.
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
"""Miss dump from an eval sidecar or in-memory rows.
|
||||
|
||||
python -m eval.trace --sidecar eval/results/baseline.jsonl
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def render_misses(rows: list[dict]) -> str:
|
||||
misses = [r for r in rows if not r.get("passed") and r.get("p_at_1", 1) == 0]
|
||||
# passed=False is the seed schema; p_at_1==0 covers pvalue sidecars
|
||||
if not misses:
|
||||
misses = [r for r in rows if not r.get("passed")]
|
||||
if not misses:
|
||||
return "# Eval misses\n\n_(none)_\n"
|
||||
lines = [f"# Eval misses ({len(misses)})", ""]
|
||||
for row in misses:
|
||||
lines += [
|
||||
f"## {row.get('query', '')}",
|
||||
"",
|
||||
f"- retriever: `{row.get('retriever')}`",
|
||||
f"- rank_first_match: `{row.get('rank_first_match')}`",
|
||||
f"- kind: `{row.get('kind', '')}`",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--sidecar", type=Path, default=Path("eval/results/baseline.jsonl"))
|
||||
p.add_argument("--misses-out", type=Path, default=Path("eval/results/misses.md"))
|
||||
args = p.parse_args()
|
||||
if not args.sidecar.exists():
|
||||
args.misses_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.misses_out.write_text("# Eval misses\n\nno sidecar\n")
|
||||
print(f"no sidecar ({args.sidecar}); wrote empty misses")
|
||||
return 0
|
||||
rows = [json.loads(line) for line in args.sidecar.read_text().splitlines() if line.strip()]
|
||||
args.misses_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.misses_out.write_text(render_misses(rows))
|
||||
print(f"wrote {args.misses_out} ({len(rows)} rows)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user