claude
d131e93252
mcp 2.x already on main. Does not replace the variety chunker (one chunk per variety is the anti-hallucination contract). - Numbered [1] citations on search_docs / search_trials - Eval JSONL sidecar, k-curve, eval.pvalue, eval.trace - Nomic prefixes at embed time only; stored text unprefixed Closes #25
51 lines
1.7 KiB
Python
51 lines
1.7 KiB
Python
"""Miss dump from an eval sidecar or in-memory rows.
|
|
|
|
python -m eval.trace --sidecar eval/results/baseline.jsonl
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def render_misses(rows: list[dict]) -> str:
|
|
misses = [r for r in rows if not r.get("passed") and r.get("p_at_1", 1) == 0]
|
|
# passed=False is the seed schema; p_at_1==0 covers pvalue sidecars
|
|
if not misses:
|
|
misses = [r for r in rows if not r.get("passed")]
|
|
if not misses:
|
|
return "# Eval misses\n\n_(none)_\n"
|
|
lines = [f"# Eval misses ({len(misses)})", ""]
|
|
for row in misses:
|
|
lines += [
|
|
f"## {row.get('query', '')}",
|
|
"",
|
|
f"- retriever: `{row.get('retriever')}`",
|
|
f"- rank_first_match: `{row.get('rank_first_match')}`",
|
|
f"- kind: `{row.get('kind', '')}`",
|
|
"",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--sidecar", type=Path, default=Path("eval/results/baseline.jsonl"))
|
|
p.add_argument("--misses-out", type=Path, default=Path("eval/results/misses.md"))
|
|
args = p.parse_args()
|
|
if not args.sidecar.exists():
|
|
args.misses_out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.misses_out.write_text("# Eval misses\n\nno sidecar\n")
|
|
print(f"no sidecar ({args.sidecar}); wrote empty misses")
|
|
return 0
|
|
rows = [json.loads(line) for line in args.sidecar.read_text().splitlines() if line.strip()]
|
|
args.misses_out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.misses_out.write_text(render_misses(rows))
|
|
print(f"wrote {args.misses_out} ({len(rows)} rows)")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|