Put the stop-word bound's evidence in the repo, not in a scratch script

COMMON_TERM_FRACTION = 0.05 is a magic number unless something prints the curve it
came from. bench/term_sweep.py asks the frozen suite as its holders would, once per
candidate bound, and writes recall and latency into the same row of
bench/reports/TERMS.md.

rare_terms now resolves its bound when the call runs instead of when the function was
defined, which is the only way a harness can move it. The default value is unchanged,
so the reports stamped at the parent commit still describe this code.

backend/src/ely_eye/text_index.py

@@ -68,7 +68,7 @@ COMMON_TERM_FRACTION = 0.05
 RAREST_KEEP = 3
 
 
-def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: float = COMMON_TERM_FRACTION) -> list[str]:
+def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: float | None = None) -> list[str]:
     """Drop the terms a library uses everywhere, and keep the ones that identify an answer.
 
     bm25 scores every row an OR query touches, so a question carrying `the` pays for a hundred
@@ -85,14 +85,17 @@ def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: fl
     counted exactly.
 
     0.05 is measured, not chosen: on the frozen suite (n=173, holder-scoped) recall@10 peaks at
-    0.899 with p50 58.6 ms there, against 0.893 / 112.9 ms at 0.20 and 0.844 / 13.4 ms at 0.01.
+    0.899 with p50 58.6 ms there, against 0.893 / 112.9 ms at 0.20 and 0.844 / 13.4 ms at 0.01, and
+    `bench/term_sweep.py` is the artifact that prints that curve. The bound is read when the call
+    runs, not when the function was written, so that sweep can move it.
     """
+    bound = COMMON_TERM_FRACTION if fraction is None else fraction
     if len(terms) == 1:
         return terms
     total = int(connection.execute("SELECT count(*) FROM atoms").fetchone()[0])
     if total == 0:
         return terms
-    cap = max(int(total * fraction) + 1, 2)
+    cap = max(int(total * bound) + 1, 2)
     counts = [(term, _term_matches(connection, fts_expression([term]), cap)) for term in terms]
     rare = [term for term, matches in counts if matches < cap]
     if rare:

bench/term_sweep.py

@@ -0,0 +1,159 @@
+"""Print the curve `COMMON_TERM_FRACTION` was picked from.
+
+A magic number is a claim until something shows the trade-off behind it. This harness asks the
+frozen suite the way its holder would, once per candidate bound, and puts recall and latency in
+the same row: keeping every word makes the query expensive, dropping too many makes it blind.
+
+Run: python -m bench.term_sweep [--probes 173]
+Writes: bench/reports/TERMS.md, bench/reports/term_sweep.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+from pathlib import Path
+from typing import Any
+
+from bench.dense_scale import copy_store
+from bench.metrics import aggregate, score_query
+from bench.query_set import load_queries, scale_probes
+from bench.report import git_commit
+from bench.run import latency
+from bench.systems import live_product_rankings, machine_facts
+
+FRACTIONS = [0.20, 0.10, 0.05, 0.02, 0.01, 0.005]
+TOP_K = 10
+DEFAULT_PROBES = 173
+DEFAULT_HOME = Path("bench/runs/baseline")
+DEFAULT_SCRATCH = Path("bench/runs/term_sweep")
+
+
+def measure(home: Path, scratch: Path, probes: int) -> dict[str, Any]:
+    """Score the shipped search path at each bound, on a copy of the frozen store."""
+    from ely_eye import text_index
+
+    copy_home, copy_seconds = copy_store(home, scratch)
+    chosen = scale_probes(load_queries(), probes)
+    shipped = text_index.COMMON_TERM_FRACTION
+    arms: list[dict[str, Any]] = []
+    try:
+        for fraction in FRACTIONS:
+            text_index.COMMON_TERM_FRACTION = fraction
+            rankings, distinct, timings = live_product_rankings(
+                copy_home,
+                [query.text for query in chosen],
+                TOP_K,
+                as_holders=[query.holder for query in chosen],
+            )
+            scores = [
+                score_query(query.query_id, ranking, set(query.gold), query.holder)
+                for query, ranking in zip(chosen, rankings, strict=True)
+            ]
+            summary = aggregate(scores)
+            timing = latency(timings)
+            arms.append(
+                {
+                    "fraction": fraction,
+                    "shipped": fraction == shipped,
+                    "recall_at_10": summary["recall_at_10"],
+                    "mrr": summary["mrr"],
+                    "hit_at_10": summary["hit_at_10"],
+                    "p50_seconds": timing["p50"],
+                    "p95_seconds": timing["p95"],
+                    "median_distinct_sources": statistics.median(distinct),
+                }
+            )
+            print(
+                f"fraction {fraction:.3f}: recall@10 {summary['recall_at_10']:.3f}, "
+                f"p50 {timing['p50'] * 1000:.1f}ms",
+                flush=True,
+            )
+    finally:
+        text_index.COMMON_TERM_FRACTION = shipped
+
+    peak = max(arms, key=lambda arm: arm["recall_at_10"])
+    return {
+        "commit": git_commit(),
+        "store": str(copy_home / "data" / "ely_eye.sqlite"),
+        "copy_seconds": copy_seconds,
+        "probes": len(chosen),
+        "shipped_fraction": shipped,
+        "peak_fraction": peak["fraction"],
+        "arms": arms,
+        "machine": machine_facts(),
+    }
+
+
+def write_markdown(report: dict[str, Any], path: Path) -> None:
+    machine = report["machine"]
+    rows = [
+        "| bound | recall@10 | MRR | hit@10 | p50 ms | p95 ms | median distinct sources |",
+        "| --- | --- | --- | --- | --- | --- | --- |",
+    ]
+    for arm in report["arms"]:
+        mark = " **(shipped)**" if arm["shipped"] else ""
+        rows.append(
+            f"| {arm['fraction']:.3f}{mark} | {arm['recall_at_10']:.3f} | {arm['mrr']:.3f} | "
+            f"{arm['hit_at_10']:.3f} | {arm['p50_seconds'] * 1000:.1f} | {arm['p95_seconds'] * 1000:.1f} | "
+            f"{arm['median_distinct_sources']} |"
+        )
+    path.write_text(
+        "\n".join(
+            [
+                "# TERMS: which query words are worth paying for",
+                "",
+                f"- commit: `{report['commit']}`",
+                f"- store: {report['store']} (copied from the frozen baseline in {report['copy_seconds']:.1f}s)",
+                (
+                    f"- {report['probes']} answerable probes, each asked as the holder the frozen set "
+                    f"assigns it, ranked by the unmodified `RetrievalService`"
+                ),
+                (
+                    f"- the bound the product ships is {report['shipped_fraction']}; the measured recall "
+                    f"peak sits at {report['peak_fraction']}"
+                ),
+                (
+                    f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, "
+                    f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"
+                ),
+                "",
+                *rows,
+                "",
+                (
+                    "The bound is a share of the library: a word carried by more atoms than that is "
+                    "dropped from the OR query, because bm25 scores every row the query touches. "
+                    "It is relative to the corpus on purpose — a corpus of PDFs and a corpus of shell "
+                    "history disagree about which of their own words are noise."
+                ),
+                "",
+                (
+                    "_Latency is wall time per search on the machine named above with the thread budget "
+                    "the benchmark pins; a run on a loaded machine is not comparable._"
+                ),
+                "",
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--home", type=Path, default=DEFAULT_HOME)
+    parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH)
+    parser.add_argument("--probes", type=int, default=DEFAULT_PROBES)
+    parser.add_argument("--out", type=Path, default=Path("bench/reports/TERMS.md"))
+    parser.add_argument("--json-out", type=Path, default=Path("bench/reports/term_sweep.json"))
+    args = parser.parse_args()
+
+    report = measure(args.home, args.scratch, args.probes)
+    args.json_out.parent.mkdir(parents=True, exist_ok=True)
+    args.json_out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    write_markdown(report, args.out)
+    print(f"wrote {args.out}")
+
+
+if __name__ == "__main__":
+    main()