morpheus-docs/eval/pvalue.py

"""Paired permutation test between two eval JSONL sidecars.

Compares per-query scores from two `eval.run_eval` sidecar files so
"P@1 went 0.88 → 0.91 on 25 queries" is not treated as a win.

    python -m eval.pvalue \\
        --a eval/results/baseline.jsonl \\
        --b eval/results/new.jsonl \\
        --metric rr

Exit 0 even when the difference is not significant — this is a report,
not a gate. No third-party deps; `random.Random(seed)` is enough.
"""
from __future__ import annotations

import argparse
import json
import random
from pathlib import Path


def load_sidecar(path: Path) -> list[dict]:
    rows: list[dict] = []
    with open(path) as fh:
        for line in fh:
            line = line.strip()
            if line:
                rows.append(json.loads(line))
    return rows


def paired_permutation(
    a: list[float],
    b: list[float],
    n_resamples: int = 10000,
    seed: int = 0,
) -> dict:
    """Two-sided paired permutation test on per-query scores.

    Null: each pair is exchangeable (randomly flipping the sign of
    A_i - B_i). p_value is the fraction of permutations whose
    |mean diff| is at least as large as the observed |mean(A-B)|.
    """
    if len(a) != len(b):
        raise ValueError(f"paired lengths differ: {len(a)} vs {len(b)}")
    if not a:
        raise ValueError("no paired queries to compare")
    diffs = [x - y for x, y in zip(a, b)]
    n = len(diffs)
    observed = sum(diffs) / n
    abs_obs = abs(observed)
    rng = random.Random(seed)
    extreme = 0
    for _ in range(n_resamples):
        total = 0.0
        for d in diffs:
            total += d if rng.random() < 0.5 else -d
        if abs(total / n) >= abs_obs - 1e-15:
            extreme += 1
    p_value = extreme / n_resamples
    return {
        "A_mean": sum(a) / n,
        "B_mean": sum(b) / n,
        "Diff(A-B)": observed,
        "p_value": p_value,
        "significant": p_value < 0.05,
        "n": n,
        "n_resamples": n_resamples,
    }


def _index(rows: list[dict], metric: str) -> dict[tuple[str, str], float]:
    """Map (retriever, query) -> score."""
    out: dict[tuple[str, str], float] = {}
    for row in rows:
        retriever = str(row.get("retriever") or "")
        query = str(row.get("query") or "")
        if metric == "p_at_1":
            score = float(row.get("p_at_1") or 0)
        else:
            score = float(row.get("rr") or 0)
        out[(retriever, query)] = score
    return out


def compare(
    rows_a: list[dict],
    rows_b: list[dict],
    metric: str = "rr",
    retriever: str | None = None,
    n_resamples: int = 10000,
    seed: int = 0,
) -> list[dict]:
    """Join on (retriever, query). One result dict per shared retriever."""
    ia, ib = _index(rows_a, metric), _index(rows_b, metric)
    retrievers = sorted({r for r, _ in ia} & {r for r, _ in ib})
    if retriever:
        retrievers = [r for r in retrievers if r == retriever]
        if not retrievers:
            raise ValueError(f"retriever {retriever!r} not in both sidecars")
    reports = []
    for name in retrievers:
        queries = sorted({q for r, q in ia if r == name} & {q for r, q in ib if r == name})
        if not queries:
            continue
        a_scores = [ia[(name, q)] for q in queries]
        b_scores = [ib[(name, q)] for q in queries]
        report = paired_permutation(a_scores, b_scores, n_resamples=n_resamples, seed=seed)
        report["retriever"] = name
        report["metric"] = metric
        reports.append(report)
    if not reports:
        raise ValueError("no overlapping (retriever, query) pairs")
    return reports


def render(reports: list[dict]) -> str:
    lines = ["# Permutation test", ""]
    for r in reports:
        sig = "yes" if r["significant"] else "no"
        lines += [
            f"## `{r['retriever']}` ({r['metric']}, n={r['n']})",
            "",
            f"- A_mean: `{r['A_mean']:.4f}`",
            f"- B_mean: `{r['B_mean']:.4f}`",
            f"- Diff(A-B): `{r['Diff(A-B)']:.4f}`",
            f"- p_value: `{r['p_value']:.4f}` ({r['n_resamples']} resamples)",
            f"- significant (p < 0.05): **{sig}**",
            "",
        ]
    return "\n".join(lines)


def main() -> int:
    p = argparse.ArgumentParser(description="Paired permutation test on two eval JSONL sidecars.")
    p.add_argument("--a", type=Path, required=True, help="sidecar JSONL (system A)")
    p.add_argument("--b", type=Path, required=True, help="sidecar JSONL (system B)")
    p.add_argument("--metric", choices=("rr", "p_at_1"), default="rr")
    p.add_argument("--retriever", default=None, help="restrict to one retriever name")
    p.add_argument("--n-resamples", type=int, default=10000)
    p.add_argument("--seed", type=int, default=0)
    args = p.parse_args()
    reports = compare(
        load_sidecar(args.a),
        load_sidecar(args.b),
        metric=args.metric,
        retriever=args.retriever,
        n_resamples=args.n_resamples,
        seed=args.seed,
    )
    print(render(reports), end="")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())