diff --git a/backend/src/ely_eye/text_index.py b/backend/src/ely_eye/text_index.py index d96f199..88b9cfb 100644 --- a/backend/src/ely_eye/text_index.py +++ b/backend/src/ely_eye/text_index.py @@ -68,7 +68,7 @@ COMMON_TERM_FRACTION = 0.05 RAREST_KEEP = 3 -def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: float = COMMON_TERM_FRACTION) -> list[str]: +def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: float | None = None) -> list[str]: """Drop the terms a library uses everywhere, and keep the ones that identify an answer. bm25 scores every row an OR query touches, so a question carrying `the` pays for a hundred @@ -85,14 +85,17 @@ def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: fl counted exactly. 0.05 is measured, not chosen: on the frozen suite (n=173, holder-scoped) recall@10 peaks at - 0.899 with p50 58.6 ms there, against 0.893 / 112.9 ms at 0.20 and 0.844 / 13.4 ms at 0.01. + 0.899 with p50 58.6 ms there, against 0.893 / 112.9 ms at 0.20 and 0.844 / 13.4 ms at 0.01, and + `bench/term_sweep.py` is the artifact that prints that curve. The bound is read when the call + runs, not when the function was written, so that sweep can move it. """ + bound = COMMON_TERM_FRACTION if fraction is None else fraction if len(terms) == 1: return terms total = int(connection.execute("SELECT count(*) FROM atoms").fetchone()[0]) if total == 0: return terms - cap = max(int(total * fraction) + 1, 2) + cap = max(int(total * bound) + 1, 2) counts = [(term, _term_matches(connection, fts_expression([term]), cap)) for term in terms] rare = [term for term, matches in counts if matches < cap] if rare: diff --git a/bench/term_sweep.py b/bench/term_sweep.py new file mode 100644 index 0000000..2a5116b --- /dev/null +++ b/bench/term_sweep.py @@ -0,0 +1,159 @@ +"""Print the curve `COMMON_TERM_FRACTION` was picked from. + +A magic number is a claim until something shows the trade-off behind it. This harness asks the +frozen suite the way its holder would, once per candidate bound, and puts recall and latency in +the same row: keeping every word makes the query expensive, dropping too many makes it blind. + +Run: python -m bench.term_sweep [--probes 173] +Writes: bench/reports/TERMS.md, bench/reports/term_sweep.json +""" + +from __future__ import annotations + +import argparse +import json +import statistics +from pathlib import Path +from typing import Any + +from bench.dense_scale import copy_store +from bench.metrics import aggregate, score_query +from bench.query_set import load_queries, scale_probes +from bench.report import git_commit +from bench.run import latency +from bench.systems import live_product_rankings, machine_facts + +FRACTIONS = [0.20, 0.10, 0.05, 0.02, 0.01, 0.005] +TOP_K = 10 +DEFAULT_PROBES = 173 +DEFAULT_HOME = Path("bench/runs/baseline") +DEFAULT_SCRATCH = Path("bench/runs/term_sweep") + + +def measure(home: Path, scratch: Path, probes: int) -> dict[str, Any]: + """Score the shipped search path at each bound, on a copy of the frozen store.""" + from ely_eye import text_index + + copy_home, copy_seconds = copy_store(home, scratch) + chosen = scale_probes(load_queries(), probes) + shipped = text_index.COMMON_TERM_FRACTION + arms: list[dict[str, Any]] = [] + try: + for fraction in FRACTIONS: + text_index.COMMON_TERM_FRACTION = fraction + rankings, distinct, timings = live_product_rankings( + copy_home, + [query.text for query in chosen], + TOP_K, + as_holders=[query.holder for query in chosen], + ) + scores = [ + score_query(query.query_id, ranking, set(query.gold), query.holder) + for query, ranking in zip(chosen, rankings, strict=True) + ] + summary = aggregate(scores) + timing = latency(timings) + arms.append( + { + "fraction": fraction, + "shipped": fraction == shipped, + "recall_at_10": summary["recall_at_10"], + "mrr": summary["mrr"], + "hit_at_10": summary["hit_at_10"], + "p50_seconds": timing["p50"], + "p95_seconds": timing["p95"], + "median_distinct_sources": statistics.median(distinct), + } + ) + print( + f"fraction {fraction:.3f}: recall@10 {summary['recall_at_10']:.3f}, " + f"p50 {timing['p50'] * 1000:.1f}ms", + flush=True, + ) + finally: + text_index.COMMON_TERM_FRACTION = shipped + + peak = max(arms, key=lambda arm: arm["recall_at_10"]) + return { + "commit": git_commit(), + "store": str(copy_home / "data" / "ely_eye.sqlite"), + "copy_seconds": copy_seconds, + "probes": len(chosen), + "shipped_fraction": shipped, + "peak_fraction": peak["fraction"], + "arms": arms, + "machine": machine_facts(), + } + + +def write_markdown(report: dict[str, Any], path: Path) -> None: + machine = report["machine"] + rows = [ + "| bound | recall@10 | MRR | hit@10 | p50 ms | p95 ms | median distinct sources |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for arm in report["arms"]: + mark = " **(shipped)**" if arm["shipped"] else "" + rows.append( + f"| {arm['fraction']:.3f}{mark} | {arm['recall_at_10']:.3f} | {arm['mrr']:.3f} | " + f"{arm['hit_at_10']:.3f} | {arm['p50_seconds'] * 1000:.1f} | {arm['p95_seconds'] * 1000:.1f} | " + f"{arm['median_distinct_sources']} |" + ) + path.write_text( + "\n".join( + [ + "# TERMS: which query words are worth paying for", + "", + f"- commit: `{report['commit']}`", + f"- store: {report['store']} (copied from the frozen baseline in {report['copy_seconds']:.1f}s)", + ( + f"- {report['probes']} answerable probes, each asked as the holder the frozen set " + f"assigns it, ranked by the unmodified `RetrievalService`" + ), + ( + f"- the bound the product ships is {report['shipped_fraction']}; the measured recall " + f"peak sits at {report['peak_fraction']}" + ), + ( + f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, " + f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}" + ), + "", + *rows, + "", + ( + "The bound is a share of the library: a word carried by more atoms than that is " + "dropped from the OR query, because bm25 scores every row the query touches. " + "It is relative to the corpus on purpose — a corpus of PDFs and a corpus of shell " + "history disagree about which of their own words are noise." + ), + "", + ( + "_Latency is wall time per search on the machine named above with the thread budget " + "the benchmark pins; a run on a loaded machine is not comparable._" + ), + "", + ] + ), + encoding="utf-8", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--home", type=Path, default=DEFAULT_HOME) + parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH) + parser.add_argument("--probes", type=int, default=DEFAULT_PROBES) + parser.add_argument("--out", type=Path, default=Path("bench/reports/TERMS.md")) + parser.add_argument("--json-out", type=Path, default=Path("bench/reports/term_sweep.json")) + args = parser.parse_args() + + report = measure(args.home, args.scratch, args.probes) + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + write_markdown(report, args.out) + print(f"wrote {args.out}") + + +if __name__ == "__main__": + main()