"""Miss dump from an eval sidecar or in-memory rows. python -m eval.trace --sidecar eval/results/baseline.jsonl """ from __future__ import annotations import argparse import json from pathlib import Path def render_misses(rows: list[dict]) -> str: misses = [r for r in rows if not r.get("passed") and r.get("p_at_1", 1) == 0] # passed=False is the seed schema; p_at_1==0 covers pvalue sidecars if not misses: misses = [r for r in rows if not r.get("passed")] if not misses: return "# Eval misses\n\n_(none)_\n" lines = [f"# Eval misses ({len(misses)})", ""] for row in misses: lines += [ f"## {row.get('query', '')}", "", f"- retriever: `{row.get('retriever')}`", f"- rank_first_match: `{row.get('rank_first_match')}`", f"- kind: `{row.get('kind', '')}`", "", ] return "\n".join(lines) def main() -> int: p = argparse.ArgumentParser() p.add_argument("--sidecar", type=Path, default=Path("eval/results/baseline.jsonl")) p.add_argument("--misses-out", type=Path, default=Path("eval/results/misses.md")) args = p.parse_args() if not args.sidecar.exists(): args.misses_out.parent.mkdir(parents=True, exist_ok=True) args.misses_out.write_text("# Eval misses\n\nno sidecar\n") print(f"no sidecar ({args.sidecar}); wrote empty misses") return 0 rows = [json.loads(line) for line in args.sidecar.read_text().splitlines() if line.strip()] args.misses_out.parent.mkdir(parents=True, exist_ok=True) args.misses_out.write_text(render_misses(rows)) print(f"wrote {args.misses_out} ({len(rows)} rows)") return 0 if __name__ == "__main__": raise SystemExit(main())