Keep every file inside the rules the repo already wrote down

The 500-line rule and the one-line-comment rule were enforced only from
memory, so they drifted. bench/filesize_check.py now checks the budget on
every tracked source file, and it passes.

Split, with behaviour proven unchanged by comparing each moved function body
against HEAD byte for byte (53 training bodies, 21 benchmark bodies, 0
differences):

  db.py        503 -> 396   store_search.py     105
  bench/run.py 609 -> 351   query_set.py         80, report.py 211
  benchmarks.py 619 -> 424  benchmark_scoring.py 262
  training.py 1630 ->  94   training_{common,encode,metrics,datasets,lora,retrieval}

TrainingService stays as a thin facade for the CLI; the facade's parameter
lists are pinned against the module functions by inspect, which is what
catches a leaked self.

Deleted the eight top-level functions no file called, after fixing the scan
that had reported 201 false positives by skipping the defining file. The
v1/v2/v3 sweep found nothing to delete and says so: every remaining version
tag is an on-disk format marker, a migration step, a fixture name, or a
third-party model name.

Collapsed 39 multi-line comment blocks (86 lines) to one line each, keeping
the measured facts and error strings that justify the code.

269 tests pass; ruff clean; mypy 24 errors in 9 files against a 24-in-6
baseline, net zero; pyright 4, the same four HEAD measures.

bench/audit.py

@@ -26,12 +26,10 @@ ROOT = Path(__file__).resolve().parent
 REPORTS = ROOT / "reports"
 CORPUS_MANIFEST = ROOT / "corpus" / "manifest.jsonl"
 
-# Words that only appear where a score is being compared to a target. A while-loop that re-rolls
-# until one of them clears is best-of-N whatever the function is called.
+# Words that only appear where a score is compared to a target; a loop that re-rolls until one clears is best-of-N whatever it is called.
 SCORE_WORDS = ("pass_rate", "pass_target", "accuracy", "recall", "hit_rate", "score", "target")
 RETRY_MARKERS = ("best_of", "bestof", "reroll", "re_roll", "until_pass", "retry_until")
-# The scorer must stay a pure function of (query, ranked ids, gold ids). Anything else and the
-# thing being graded starts grading itself.
+# The scorer must stay a pure function of (query, ranked ids, gold ids), or the thing being graded starts grading itself.
 SCORER_FORBIDDEN = ("ely_eye", "transformers", "sentence_transformers", "torch", "bench.systems", "bench.counterpart")
 FROZEN_REPORTS = ("baseline.json", "baseline_external.json")
 

bench/build_code.py

@@ -124,8 +124,7 @@ def build_functions() -> tuple[list, list[dict[str, Any]], dict]:
         if not stripped.strip():
             skipped["empty_after_strip"] += 1
             continue
-        # One dataset row is one function; its index is the only thing that makes the
-        # identity unique, because a file may define the same name twice.
+        # One dataset row is one function; its index is the only unique identity, because a file may define the same name twice.
         tag = hashlib.sha256(f"{func_path}#{func_name}#{index}".encode()).hexdigest()[:10]
         relative = function_path(repository, func_path, func_name, tag)
         target = DATA_ROOT / relative

bench/build_visrag.py

@@ -74,8 +74,7 @@ def image_name(corpus_id: str, raw: bytes) -> str:
             stem = corpus_id[: -len(known)]
             break
     if len(stem.encode()) > 120 or safe_segment(stem, f"corpus-id {corpus_id!r}") != stem:
-        # Any character we would have to rewrite means the id is not filesystem-safe:
-        # fall back to a digest so two ids can never land on the same file.
+        # Any character we would have to rewrite means the id is not filesystem-safe, so fall back to a digest that cannot collide.
         stem = hashlib.sha256(corpus_id.encode()).hexdigest()[:20]
     return stem + suffix
 

bench/ingest_baseline.py

@@ -35,9 +35,7 @@ REPO = Path(__file__).resolve().parent.parent
 RUNS = REPO / "bench" / "runs" / "baseline"
 SHARDS = RUNS / "shards"
 TABLES = ("sources", "atoms", "atom_fts", "cartridges", "cache_events", "evidence_edges")
-# Virtual-address ceiling per worker. ONNX Runtime and torch reserve far more address space than
-# they touch, so a 3 GiB ceiling killed OCR on large images ("Unable to allocate 582. MiB for an
-# array with shape (5632, 9024, 3)") even though resident memory stayed near 1.5 GiB.
+# ONNX Runtime and torch reserve far more address space than they touch, so a 3 GiB ceiling killed OCR on large images ("Unable to allocate 582. MiB for an array with shape (5632, 9024, 3)") while resident memory stayed near 1.5 GiB.
 WORKER_MEMORY_BYTES = 6 * 1024**3
 MAX_POOL_ATTEMPTS = 5
 DELETE_STATEMENTS = (
@@ -125,8 +123,7 @@ def _pin_inference_threads() -> None:
         options = ort.SessionOptions()
         options.intra_op_num_threads = 1
         options.inter_op_num_threads = 1
-        # The CPU arena keeps freed blocks mapped, so a worker that OCRs thousands of pages grows its
-        # resident set until it hits its ceiling on an 11 MiB array. Hand the memory back instead.
+        # The CPU arena keeps freed blocks mapped, so a worker OCRing thousands of pages grows until it hits its ceiling on an 11 MiB array.
         options.enable_cpu_mem_arena = False
         kwargs["sess_options"] = options
         return original(model_path, *args, **kwargs)
@@ -197,8 +194,7 @@ def ingest_shard(job: tuple[int, list[Path]]) -> dict:
         with ledger.open("a", encoding="utf-8") as handle:
             handle.write(json.dumps(outcome) + "\n")
         if ingested % 25 == 0:
-            # ONNX Runtime keeps the address space it has touched, and one shard runs thousands of
-            # files, so the arena has to be handed back or the worker hits its ceiling on an 11 MiB array.
+            # ONNX Runtime keeps the address space it touched, so one shard of thousands of files has to hand the arena back.
             gc.collect()
             elapsed = time.monotonic() - started
             print(f"shard-{index:02d} {ingested}/{len(paths)} files in {elapsed / 60:.1f} min", flush=True)
@@ -310,9 +306,7 @@ def merge() -> dict:
                     copied_objects += 1
     connection.commit()
 
-    # A file processed in two attempts lands in two shards. The atoms table
-    # dedupes on atom_id; fts5 has no key and kept every copy (288,266 rows for
-    # 165,973 atoms), which would inflate the product's text search statistics.
+    # A file processed twice lands in two shards: atoms dedupe on atom_id but fts5 has no key and kept every copy (288,266 rows for 165,973 atoms).
     connection.execute("DELETE FROM atom_fts")
     connection.execute(
         "INSERT INTO atom_fts (atom_id, source, text) SELECT atom_id, source, text FROM atoms"
@@ -335,8 +329,7 @@ def merge() -> dict:
         raise SystemExit(f"{main_db}: {len(unmapped)} sources are not named by the corpus index, first {unmapped[:3]}")
     product_paths = {path for path, _ in rows}
     product_ids = {source_id for _, source_id in rows}
-    # The product names a source by content hash, so two byte-identical files
-    # collapse into one source row (500 duplicate LongMemEval sessions).
+    # The product names a source by content hash, so byte-identical files collapse into one source row (500 duplicate LongMemEval sessions).
     collapsed_input = sorted(path for path in done - product_paths if safe_source_id(Path(path), sha256_bytes(Path(path).read_bytes())) in product_ids)
     unaccounted = sorted(done - product_paths - set(collapsed_input))
     if unaccounted:
@@ -363,8 +356,7 @@ def merge() -> dict:
         )
     counts["uncovered_sources"] = len(total - covered)
     if counts["files_not_ingested"] == 0 and covered != total:
-        # Two honest reasons: byte-identical inputs collapse into one content-hashed
-        # source, and files the product cannot ingest at all. Anything else is a bug.
+        # Two honest reasons: byte-identical inputs collapse into one content-hashed source, and files the product cannot ingest at all.
         explained = counts["duplicate_content_files"] + counts["files_failed"]
         if counts["uncovered_sources"] > explained:
             raise SystemExit(

bench/query_set.py

@@ -0,0 +1,83 @@
+"""The frozen query set: what is asked, who asks it, and which probes to time."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from bench.sets_common import MODALITIES, SETS, read_jsonl
+
+
+@dataclass(frozen=True)
+class Query:
+    query_id: str
+    text: str
+    modality: str
+    gold: frozenset[str]
+    holder: str | None
+    answerable: bool
+    as_of: str | None
+
+
+
+def load_queries(modalities: Sequence[str] = MODALITIES) -> list[Query]:
+    """Every query the suite froze, in file order."""
+    queries: list[Query] = []
+    for modality in modalities:
+        path = SETS / f"{modality}.jsonl"
+        if not path.is_file():
+            raise SystemExit(f"{path}: missing set, run python -m bench.build_sets first")
+        for record in read_jsonl(path):
+            gold = frozenset(hit["source_id"] for hit in record["gold"])
+            answerable = not bool(record["unanswerable"])
+            if answerable != bool(gold):
+                raise SystemExit(
+                    f"{path}: {record['query_id']} carries {len(gold)} gold hits but "
+                    f"unanswerable={record['unanswerable']}"
+                )
+            queries.append(
+                Query(
+                    query_id=record["query_id"],
+                    text=record["question"],
+                    modality=modality,
+                    gold=gold,
+                    holder=record["scope"]["holder"],
+                    answerable=answerable,
+                    as_of=record.get("as_of"),
+                )
+            )
+    if not queries:
+        raise SystemExit("no queries loaded")
+    return queries
+
+
+
+def holder_index(modalities: Sequence[str] = MODALITIES) -> dict[str, str]:
+    """source_id -> holder, from the corpus side of the suite."""
+    holders: dict[str, str] = {}
+    for modality in modalities:
+        path = SETS / "corpus" / f"{modality}.jsonl"
+        if not path.is_file():
+            raise SystemExit(f"{path}: missing corpus index, run python -m bench.build_sets first")
+        for record in read_jsonl(path):
+            holders[record["source_id"]] = record["holder"]
+    return holders
+
+
+
+SCALE_PROBES = 40
+
+
+def scale_probes(queries: Sequence[Query], count: int = SCALE_PROBES) -> list[Query]:
+    """Queries to time at every corpus size.
+
+    Latency does not need gold hits, and a truncated corpus keeps almost none of them,
+    so selecting probes by gold would silently drop the small sizes entirely.
+    """
+    if count <= 0:
+        raise SystemExit("the scale curve needs at least one probe query")
+    answerable = [query for query in queries if query.answerable]
+    if not answerable:
+        raise SystemExit("the scale curve needs answerable queries")
+    stride = max(1, len(answerable) // count)
+    return answerable[::stride][:count]

bench/report.py

@@ -0,0 +1,211 @@
+"""The report the ablation leaves behind: metadata, the chance floor, and the markdown table."""
+
+from __future__ import annotations
+
+import json
+import platform
+import subprocess
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from bench.metrics import chance_recall_at_k
+from bench.query_set import Query
+from bench.sets_common import MODALITIES, ROOT, SETS, sha256_file
+from bench.systems import DENSE_MODEL, RERANK_MODEL, machine_facts
+
+
+def chance_row(queries: Sequence[Query], corpus_sources: int) -> dict[str, float]:
+    """What a random ranking would score on the same queries."""
+    answerable = [query for query in queries if query.answerable]
+    if not answerable:
+        raise SystemExit("chance needs answerable queries")
+    return {
+        "corpus_sources": corpus_sources,
+        "recall_at_1": sum(chance_recall_at_k(len(query.gold), corpus_sources, 1) for query in answerable) / len(answerable),
+        "recall_at_5": sum(chance_recall_at_k(len(query.gold), corpus_sources, 5) for query in answerable) / len(answerable),
+        "recall_at_10": sum(chance_recall_at_k(len(query.gold), corpus_sources, 10) for query in answerable) / len(answerable),
+    }
+
+
+
+def git_commit() -> str:
+    result = subprocess.run(
+        ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True, check=True, timeout=30
+    )
+    commit = result.stdout.strip()
+    if len(commit) != 40:
+        raise SystemExit(f"git rev-parse HEAD returned {commit!r}, expected a 40 character sha")
+    return commit
+
+
+
+def store_schema_version(store: Path) -> int:
+    """The schema the measured store actually carries, read from the file."""
+    import sqlite3
+
+    if not store.exists():
+        raise SystemExit(f"{store}: the measured store is missing")
+    connection = sqlite3.connect(f"file:{store}?mode=ro", uri=True)
+    try:
+        row = connection.execute("PRAGMA user_version").fetchone()
+    finally:
+        connection.close()
+    return int(row[0])
+
+
+
+def meta(
+    corpus_sources: int, queries: Sequence[Query], limit: int | None = None, store: Path | None = None
+) -> dict[str, Any]:
+    manifest_path = SETS / "manifest.json"
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise SystemExit(f"{manifest_path}: cannot read the frozen set manifest: {error}") from error
+    return {
+        "generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "git_commit": git_commit(),
+        "python": platform.python_version(),
+        "machine": machine_facts(),
+        "corpus_manifest_sha256": manifest.get("corpus_manifest_sha256"),
+        "sets": {name: sha256_file(SETS / f"{name}.jsonl") for name in MODALITIES},
+        "queries": len(queries),
+        "answerable": sum(1 for query in queries if query.answerable),
+        "limit_per_modality": limit,
+        "corpus_sources": corpus_sources,
+        "store_schema_version_at_start": None if store is None else store_schema_version(store),
+        "models": {"dense": DENSE_MODEL, "rerank": RERANK_MODEL},
+    }
+
+
+
+def scale_arm_names(rows: Sequence[dict[str, Any]]) -> list[str]:
+    """Arms a scale row reports: only keys whose value is a latency dict, in order.
+
+    A scale row also carries strings (the note explaining what the row means), and
+    treating those as arms is how the report writer first crashed.
+    """
+    names: list[str] = []
+    for row in rows:
+        for name, value in row.items():
+            if isinstance(value, dict) and name not in names:
+                names.append(name)
+    if not names:
+        raise SystemExit("scale rows report no measured arm")
+    return names
+
+
+
+def write_markdown(report: dict[str, Any], path: Path, name: str = "BASELINE") -> None:
+    """Human-readable twin of the JSON report."""
+    schema = report["meta"].get("store_schema_version_at_start")
+    lines = [
+        f"# {name}: what the product retrieves at this build",
+        "",
+        f"- generated {report['meta']['generated_at']} at commit {report['meta']['git_commit'][:12]}",
+        ("" if schema is None else f"- measured store schema: v{schema}"),
+        (
+            f"- corpus {report['meta']['corpus_sources']} sources, {report['meta']['queries']} queries "
+            f"({report['meta']['answerable']} answerable)"
+        ),
+        f"- models: dense `{report['meta']['models']['dense']}`, rerank `{report['meta']['models']['rerank']}`",
+        (
+            f"- machine: {report['meta']['machine']['cpus']} cores, "
+            f"{report['meta']['machine']['threads_per_query']} threads per query, load average "
+            f"{report['meta']['machine']['load_average_1_5_15'][0]:.1f} when the report was written"
+        ),
+        "",
+        "| arm | recall@1 | recall@5 | recall@10 | nDCG@10 | MRR | hit@10 (95% lower) | p50 s | p95 s |",
+        "|---|---|---|---|---|---|---|---|---|",
+    ]
+    chance = report["chance"]
+    lines.append(
+        f"| chance (random ranking) | {chance['recall_at_1']:.3f} | {chance['recall_at_5']:.3f} "
+        f"| {chance['recall_at_10']:.3f} | - | - | - | - | - |"
+    )
+    if note := next((arm["latency_seconds"].get("note") for arm in report["arms"] if arm["latency_seconds"].get("note")), None):
+        lines += ["", f"_{note} (n={report['arms'][0]['latency_seconds']['queries']} queries)_"]
+    for arm in report["arms"]:
+        overall = arm["overall"]
+        latency = arm["latency_seconds"]
+        lines.append(
+            f"| {arm['arm']} | {overall['recall_at_1']:.3f} | {overall['recall_at_5']:.3f} "
+            f"| {overall['recall_at_10']:.3f} | {overall['ndcg_at_10']:.3f} | {overall['mrr']:.3f} "
+            f"| {overall['hit_at_10']:.3f} ({overall['hit_at_10_wilson_lower']:.3f}) "
+            f"| {latency['p50']:.3f} | {latency['p95']:.3f} |"
+        )
+    shown = [modality for modality in MODALITIES if any(modality in arm["by_modality"] for arm in report["arms"])]
+    if not shown:
+        raise SystemExit("no per-modality scores to write")
+    lines += ["", "## recall@10 by modality", "", "| arm | " + " | ".join(shown) + " |", "|---|" + "---|" * len(shown)]
+    for arm in report["arms"]:
+        cells = [
+            f"{arm['by_modality'][modality]['recall_at_10']:.3f}" if modality in arm["by_modality"] else "-"
+            for modality in shown
+        ]
+        lines.append(f"| {arm['arm']} | " + " | ".join(cells) + " |")
+    total = report["meta"]["queries"]
+    if total <= 0:
+        raise SystemExit("the report records no queries")
+    share = 1 - report["meta"]["answerable"] / total
+    lines += ["", f"## unanswerable queries ({share:.1%} of the suite)", ""]
+    for arm in report["arms"]:
+        overall = arm["overall"]
+        lines.append(
+            f"- {arm['arm']}: abstains on {overall['abstention_rate_unanswerable']:.0%} of unanswerable queries, "
+            f"and its top hit comes from the query's own holder for {overall['top_hit_from_holder_unanswerable']:.0%} of them; "
+            "the rest are confident answers pulled out of somebody else's memory"
+        )
+    if "live_product" in report:
+        live = report["live_product"]
+        lines += [
+            "",
+            "## the product path, asked as its holder",
+            "",
+            (
+                f"- RetrievalService.search as the query's own holder: recall@10 {live['recall_at_10']:.3f}, "
+                f"MRR {live['mrr']:.3f}, hit@10 {live['hit_at_10']:.3f} (95% lower {live['hit_at_10_wilson_lower']:.3f}) "
+                f"over n={live['queries']} answerable queries"
+            ),
+            (
+                f"- latency p50 {live['p50']:.3f}s, p95 {live['p95']:.3f}s, mean {live['mean']:.3f}s, "
+                f"max {live['max']:.3f}s"
+            ),
+            (
+                f"- top-10 overlap with the fast BM25 path: {live['top10_overlap_with_bm25_only']:.1%}, "
+                f"identical order {live['identical_top10']:.1%}"
+            ),
+            (
+                f"- its top-{live['top_k']} atoms cover "
+                f"{live['distinct_sources_in_top10']['mean']:.1f} distinct sources on average "
+                f"(min {live['distinct_sources_in_top10']['min']}): the product ranks atoms and never diversifies, "
+                "so ten slots can fill with fragments of one document"
+            ),
+        ]
+    if report.get("scale"):
+        measured = scale_arm_names(report["scale"])
+        header = " | ".join(f"{arm} p50/p95 s" for arm in measured)
+        lines += ["", "## latency vs corpus size", "", f"| sources | atoms | {header} |", "|" + "---|" * (2 + len(measured))]
+        for row in report["scale"]:
+            cells = [f"{row[arm]['p50']:.3f}/{row[arm]['p95']:.3f}" for arm in measured]
+            lines.append(f"| {row['sources']} | {row['atoms']} | " + " | ".join(cells) + " |")
+        note = report["scale"][0].get("note")
+        if note:
+            lines += ["", f"_{note}_"]
+        load = report["meta"]["machine"]["load_average_1_5_15"][0]
+        lines.append(
+            f"\n_latency was measured at load average {load:.1f} on {report['meta']['machine']['cpus']} cores; "
+            "a busy machine inflates these figures, so read them as a ceiling_"
+        )
+    lines += [
+        "",
+        "## reading",
+        "",
+        "- recall counts a source as found only when it is the source a third party marked correct.",
+        "- abstention is the only honest number for unanswerable queries: a retriever that always",
+        "  returns ten results never says it does not know.",
+        "- chance is the floor any of these arms must beat; a number near chance is not a capability.",
+    ]
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")

bench/run.py

@@ -1,132 +1,36 @@
-"""Run the retrieval ablation over the frozen corpus and write the baseline report.
-
-Five arms, one database: sparse only, dense only, graph only, the product's weighted
-fusion, and fusion plus a cross-encoder reranker. Every number in the report is either
-measured or explicitly marked as chance.
-
-Run: python -m bench.run [--arms bm25_only] [--limit 200] [--no-live] [--out bench/reports]
-"""
+"""Run the retrieval ablation over the frozen corpus and write the report: python -m bench.run [--arms bm25_only] [--limit 200] [--no-live]."""
 
 from __future__ import annotations
 
 import argparse
 import hashlib
 import json
-import platform
-import subprocess
 from collections.abc import Sequence
-from dataclasses import dataclass
-from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
 import numpy as np
 
-from bench.metrics import aggregate, chance_recall_at_k, score_query
-from bench.sets_common import MODALITIES, SETS, read_jsonl, sha256_file
+from bench.metrics import aggregate, score_query
+from bench.query_set import Query, holder_index, load_queries, scale_probes
+from bench.report import chance_row, meta, write_markdown
+from bench.sets_common import MODALITIES, ROOT
 from bench.systems import (
     ARMS,
-    DENSE_MODEL,
     MODELS,
-    RERANK_MODEL,
     ArmRunner,
     Atom,
     JointRunner,
     arm_needs,
     live_product_rankings,
     load_atoms,
-    machine_facts,
 )
 
-ROOT = SETS.parent
 REPORTS = ROOT / "reports"
 DB_HOME = ROOT / "runs" / "baseline"
 LIVE_QUERIES = 300  # the acceptance bar names n>=300; the SQL path answers in well under a second
 PROGRESS_EVERY = 250  # an arm takes tens of minutes over 5,308 queries; silence looks like a hang
 SCALE_SIZES = (500, 2_000, 10_000, 50_000)
-SCALE_PROBES = 40
-
-
-@dataclass(frozen=True)
-class Query:
-    query_id: str
-    text: str
-    modality: str
-    gold: frozenset[str]
-    holder: str | None
-    answerable: bool
-    as_of: str | None
-
-
-def load_queries(modalities: Sequence[str] = MODALITIES) -> list[Query]:
-    """Every query the suite froze, in file order."""
-    queries: list[Query] = []
-    for modality in modalities:
-        path = SETS / f"{modality}.jsonl"
-        if not path.is_file():
-            raise SystemExit(f"{path}: missing set, run python -m bench.build_sets first")
-        for record in read_jsonl(path):
-            gold = frozenset(hit["source_id"] for hit in record["gold"])
-            answerable = not bool(record["unanswerable"])
-            if answerable != bool(gold):
-                raise SystemExit(
-                    f"{path}: {record['query_id']} carries {len(gold)} gold hits but "
-                    f"unanswerable={record['unanswerable']}"
-                )
-            queries.append(
-                Query(
-                    query_id=record["query_id"],
-                    text=record["question"],
-                    modality=modality,
-                    gold=gold,
-                    holder=record["scope"]["holder"],
-                    answerable=answerable,
-                    as_of=record.get("as_of"),
-                )
-            )
-    if not queries:
-        raise SystemExit("no queries loaded")
-    return queries
-
-
-def holder_index(modalities: Sequence[str] = MODALITIES) -> dict[str, str]:
-    """source_id -> holder, from the corpus side of the suite."""
-    holders: dict[str, str] = {}
-    for modality in modalities:
-        path = SETS / "corpus" / f"{modality}.jsonl"
-        if not path.is_file():
-            raise SystemExit(f"{path}: missing corpus index, run python -m bench.build_sets first")
-        for record in read_jsonl(path):
-            holders[record["source_id"]] = record["holder"]
-    return holders
-
-
-def chance_row(queries: Sequence[Query], corpus_sources: int) -> dict[str, float]:
-    """What a random ranking would score on the same queries."""
-    answerable = [query for query in queries if query.answerable]
-    if not answerable:
-        raise SystemExit("chance needs answerable queries")
-    return {
-        "corpus_sources": corpus_sources,
-        "recall_at_1": sum(chance_recall_at_k(len(query.gold), corpus_sources, 1) for query in answerable) / len(answerable),
-        "recall_at_5": sum(chance_recall_at_k(len(query.gold), corpus_sources, 5) for query in answerable) / len(answerable),
-        "recall_at_10": sum(chance_recall_at_k(len(query.gold), corpus_sources, 10) for query in answerable) / len(answerable),
-    }
-
-
-def scale_probes(queries: Sequence[Query], count: int = SCALE_PROBES) -> list[Query]:
-    """Queries to time at every corpus size.
-
-    Latency does not need gold hits, and a truncated corpus keeps almost none of them,
-    so selecting probes by gold would silently drop the small sizes entirely.
-    """
-    if count <= 0:
-        raise SystemExit("the scale curve needs at least one probe query")
-    answerable = [query for query in queries if query.answerable]
-    if not answerable:
-        raise SystemExit("the scale curve needs answerable queries")
-    stride = max(1, len(answerable) // count)
-    return answerable[::stride][:count]
 
 
 def scale_curve(queries: Sequence[Query], atoms: Sequence[Atom], arms: Sequence[str]) -> list[dict[str, Any]]:
@@ -209,183 +113,6 @@ def live_check(
     return metrics
 
 
-def git_commit() -> str:
-    result = subprocess.run(
-        ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True, check=True, timeout=30
-    )
-    commit = result.stdout.strip()
-    if len(commit) != 40:
-        raise SystemExit(f"git rev-parse HEAD returned {commit!r}, expected a 40 character sha")
-    return commit
-
-
-def store_schema_version(store: Path) -> int:
-    """The schema the measured store actually carries, read from the file."""
-    import sqlite3
-
-    if not store.exists():
-        raise SystemExit(f"{store}: the measured store is missing")
-    connection = sqlite3.connect(f"file:{store}?mode=ro", uri=True)
-    try:
-        row = connection.execute("PRAGMA user_version").fetchone()
-    finally:
-        connection.close()
-    return int(row[0])
-
-
-def meta(
-    corpus_sources: int, queries: Sequence[Query], limit: int | None = None, store: Path | None = None
-) -> dict[str, Any]:
-    manifest_path = SETS / "manifest.json"
-    try:
-        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError) as error:
-        raise SystemExit(f"{manifest_path}: cannot read the frozen set manifest: {error}") from error
-    return {
-        "generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
-        "git_commit": git_commit(),
-        "python": platform.python_version(),
-        "machine": machine_facts(),
-        "corpus_manifest_sha256": manifest.get("corpus_manifest_sha256"),
-        "sets": {name: sha256_file(SETS / f"{name}.jsonl") for name in MODALITIES},
-        "queries": len(queries),
-        "answerable": sum(1 for query in queries if query.answerable),
-        "limit_per_modality": limit,
-        "corpus_sources": corpus_sources,
-        "store_schema_version_at_start": None if store is None else store_schema_version(store),
-        "models": {"dense": DENSE_MODEL, "rerank": RERANK_MODEL},
-    }
-
-
-def scale_arm_names(rows: Sequence[dict[str, Any]]) -> list[str]:
-    """Arms a scale row reports: only keys whose value is a latency dict, in order.
-
-    A scale row also carries strings (the note explaining what the row means), and
-    treating those as arms is how the report writer first crashed.
-    """
-    names: list[str] = []
-    for row in rows:
-        for name, value in row.items():
-            if isinstance(value, dict) and name not in names:
-                names.append(name)
-    if not names:
-        raise SystemExit("scale rows report no measured arm")
-    return names
-
-
-def write_markdown(report: dict[str, Any], path: Path, name: str = "BASELINE") -> None:
-    """Human-readable twin of the JSON report."""
-    schema = report["meta"].get("store_schema_version_at_start")
-    lines = [
-        f"# {name}: what the product retrieves at this build",
-        "",
-        f"- generated {report['meta']['generated_at']} at commit {report['meta']['git_commit'][:12]}",
-        ("" if schema is None else f"- measured store schema: v{schema}"),
-        (
-            f"- corpus {report['meta']['corpus_sources']} sources, {report['meta']['queries']} queries "
-            f"({report['meta']['answerable']} answerable)"
-        ),
-        f"- models: dense `{report['meta']['models']['dense']}`, rerank `{report['meta']['models']['rerank']}`",
-        (
-            f"- machine: {report['meta']['machine']['cpus']} cores, "
-            f"{report['meta']['machine']['threads_per_query']} threads per query, load average "
-            f"{report['meta']['machine']['load_average_1_5_15'][0]:.1f} when the report was written"
-        ),
-        "",
-        "| arm | recall@1 | recall@5 | recall@10 | nDCG@10 | MRR | hit@10 (95% lower) | p50 s | p95 s |",
-        "|---|---|---|---|---|---|---|---|---|",
-    ]
-    chance = report["chance"]
-    lines.append(
-        f"| chance (random ranking) | {chance['recall_at_1']:.3f} | {chance['recall_at_5']:.3f} "
-        f"| {chance['recall_at_10']:.3f} | - | - | - | - | - |"
-    )
-    if note := next((arm["latency_seconds"].get("note") for arm in report["arms"] if arm["latency_seconds"].get("note")), None):
-        lines += ["", f"_{note} (n={report['arms'][0]['latency_seconds']['queries']} queries)_"]
-    for arm in report["arms"]:
-        overall = arm["overall"]
-        latency = arm["latency_seconds"]
-        lines.append(
-            f"| {arm['arm']} | {overall['recall_at_1']:.3f} | {overall['recall_at_5']:.3f} "
-            f"| {overall['recall_at_10']:.3f} | {overall['ndcg_at_10']:.3f} | {overall['mrr']:.3f} "
-            f"| {overall['hit_at_10']:.3f} ({overall['hit_at_10_wilson_lower']:.3f}) "
-            f"| {latency['p50']:.3f} | {latency['p95']:.3f} |"
-        )
-    shown = [modality for modality in MODALITIES if any(modality in arm["by_modality"] for arm in report["arms"])]
-    if not shown:
-        raise SystemExit("no per-modality scores to write")
-    lines += ["", "## recall@10 by modality", "", "| arm | " + " | ".join(shown) + " |", "|---|" + "---|" * len(shown)]
-    for arm in report["arms"]:
-        cells = [
-            f"{arm['by_modality'][modality]['recall_at_10']:.3f}" if modality in arm["by_modality"] else "-"
-            for modality in shown
-        ]
-        lines.append(f"| {arm['arm']} | " + " | ".join(cells) + " |")
-    total = report["meta"]["queries"]
-    if total <= 0:
-        raise SystemExit("the report records no queries")
-    share = 1 - report["meta"]["answerable"] / total
-    lines += ["", f"## unanswerable queries ({share:.1%} of the suite)", ""]
-    for arm in report["arms"]:
-        overall = arm["overall"]
-        lines.append(
-            f"- {arm['arm']}: abstains on {overall['abstention_rate_unanswerable']:.0%} of unanswerable queries, "
-            f"and its top hit comes from the query's own holder for {overall['top_hit_from_holder_unanswerable']:.0%} of them; "
-            "the rest are confident answers pulled out of somebody else's memory"
-        )
-    if "live_product" in report:
-        live = report["live_product"]
-        lines += [
-            "",
-            "## the product path, asked as its holder",
-            "",
-            (
-                f"- RetrievalService.search as the query's own holder: recall@10 {live['recall_at_10']:.3f}, "
-                f"MRR {live['mrr']:.3f}, hit@10 {live['hit_at_10']:.3f} (95% lower {live['hit_at_10_wilson_lower']:.3f}) "
-                f"over n={live['queries']} answerable queries"
-            ),
-            (
-                f"- latency p50 {live['p50']:.3f}s, p95 {live['p95']:.3f}s, mean {live['mean']:.3f}s, "
-                f"max {live['max']:.3f}s"
-            ),
-            (
-                f"- top-10 overlap with the fast BM25 path: {live['top10_overlap_with_bm25_only']:.1%}, "
-                f"identical order {live['identical_top10']:.1%}"
-            ),
-            (
-                f"- its top-{live['top_k']} atoms cover "
-                f"{live['distinct_sources_in_top10']['mean']:.1f} distinct sources on average "
-                f"(min {live['distinct_sources_in_top10']['min']}): the product ranks atoms and never diversifies, "
-                "so ten slots can fill with fragments of one document"
-            ),
-        ]
-    if report.get("scale"):
-        measured = scale_arm_names(report["scale"])
-        header = " | ".join(f"{arm} p50/p95 s" for arm in measured)
-        lines += ["", "## latency vs corpus size", "", f"| sources | atoms | {header} |", "|" + "---|" * (2 + len(measured))]
-        for row in report["scale"]:
-            cells = [f"{row[arm]['p50']:.3f}/{row[arm]['p95']:.3f}" for arm in measured]
-            lines.append(f"| {row['sources']} | {row['atoms']} | " + " | ".join(cells) + " |")
-        note = report["scale"][0].get("note")
-        if note:
-            lines += ["", f"_{note}_"]
-        load = report["meta"]["machine"]["load_average_1_5_15"][0]
-        lines.append(
-            f"\n_latency was measured at load average {load:.1f} on {report['meta']['machine']['cpus']} cores; "
-            "a busy machine inflates these figures, so read them as a ceiling_"
-        )
-    lines += [
-        "",
-        "## reading",
-        "",
-        "- recall counts a source as found only when it is the source a third party marked correct.",
-        "- abstention is the only honest number for unanswerable queries: a retriever that always",
-        "  returns ten results never says it does not know.",
-        "- chance is the floor any of these arms must beat; a number near chance is not a capability.",
-    ]
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-
-
 def load_config(parser: argparse.ArgumentParser, args: argparse.Namespace, path: Path) -> None:
     """Fill in whatever the command line left at its default; an explicit flag still wins."""
     try:
@@ -514,8 +241,7 @@ def main() -> int:
         load_config(parser, args, args.config)
 
     modalities = args.modality or list(MODALITIES)
-    # action="extend" appends to the default, so an explicit --arms would otherwise
-    # run every arm plus the requested ones.
+    # action="extend" appends to the default, so an explicit --arms would otherwise run every arm plus the requested ones.
     arms_requested: list[str] = list(args.arms or ARMS)
     queries = load_queries(modalities)
     if args.limit:
@@ -533,8 +259,7 @@ def main() -> int:
         "meta": meta(corpus_sources, queries, args.limit, store=DB_HOME / "data" / "ely_eye.sqlite"),
         "chance": chance_row(queries, corpus_sources),
     }
-    # One pass over the queries: each signal is computed once and every arm is derived
-    # from it, and each query is checkpointed so a kill costs one query, not an arm.
+    # One pass over the queries: each signal is computed once, every arm derives from it, and each query is checkpointed so a kill costs one query.
     mark = fingerprint(arms_requested, queries, atoms)
     rankings_path = DB_HOME / "rankings.jsonl"
     done = load_rankings(rankings_path, mark)

bench/run_external.py

@@ -27,7 +27,8 @@ from bench.counterpart import (
 )
 from bench.ingest_baseline import RUNS, database_path
 from bench.metrics import aggregate, chance_recall_at_k, score_query
-from bench.run import Query, git_commit, holder_index, load_queries
+from bench.query_set import Query, holder_index, load_queries
+from bench.report import git_commit
 from bench.sets_common import ROOT
 from bench.systems import machine_facts
 

bench/scale_check.py

@@ -14,7 +14,8 @@ import time
 from pathlib import Path
 from typing import Any
 
-from bench.run import latency, load_queries, scale_probes
+from bench.query_set import load_queries, scale_probes
+from bench.run import latency
 from bench.systems import machine_facts
 
 DEFAULT_HOME = Path("bench/runs/baseline")

bench/sets_common.py

@@ -13,8 +13,7 @@ from typing import Any
 ROOT = Path(__file__).resolve().parent
 CORPUS = ROOT / "corpus"
 SETS = ROOT / "sets"
-# Derived media (images pulled out of parquet, video frames, crops) lives outside
-# the repository: tens of thousands of new files inside it swamps repo tooling.
+# Derived media (parquet images, video frames, crops) lives outside the repository: tens of thousands of new files inside it swamps repo tooling.
 DATA_ROOT = Path(os.environ.get("ELY_EYE_BENCH_DATA", str(Path.home() / "ely_eye_bench_data")))
 CORPUS_ROOTS = {"corpus": CORPUS, "derived": DATA_ROOT}
 MODALITIES = ("pdf", "image", "ui_screenshot", "video", "code", "conversation")

bench/systems.py

@@ -324,8 +324,7 @@ class ArmRunner:
         return float(np.percentile(np.asarray(self.timings), fraction * 100))
 
 
-# Which measured signals each arm consumes; JointRunner times each signal once and
-# attributes it to every arm that uses it.
+# Which measured signals each arm consumes; JointRunner times each signal once and attributes it to every arm that uses it.
 ARM_COMPONENTS: dict[str, tuple[str, ...]] = {
     "bm25_only": ("sparse",),
     "dense_only": ("dense",),

bench/tests/test_joint.py

@@ -16,16 +16,9 @@ import numpy as np
 import pytest
 
 from bench import systems
-from bench.run import (
-    Query,
-    fingerprint,
-    latency,
-    load_rankings,
-    remeasure_latency,
-    scale_arm_names,
-    scale_probes,
-    write_markdown,
-)
+from bench.query_set import Query, scale_probes
+from bench.report import scale_arm_names, write_markdown
+from bench.run import fingerprint, latency, load_rankings, remeasure_latency
 from bench.systems import ARMS, ArmRunner, Atom, JointRunner
 from bench.tests.test_systems import ATOMS
 

bench/tests/test_systems.py

@@ -27,8 +27,7 @@ ATOMS = [
     Atom(atom_id="a3", source_id="src_red", text="the key hangs on the red door"),
 ]
 
-# The product names a source by content hash; the benchmark names the same file by
-# its corpus id (the source_id above). load_atoms joins the two through sources.path.
+# The product names a source by content hash, the benchmark by corpus id; load_atoms joins the two through sources.path.
 PRODUCT_SOURCES = {"prod_red_9f3a": "/fixture/red.png", "prod_blue_1c77": "/fixture/blue.png"}
 ATOM_PRODUCT_IDS = ["prod_red_9f3a", "prod_blue_1c77", "prod_red_9f3a"]
 
@@ -123,8 +122,7 @@ def test_load_atoms_returns_every_row_in_a_stable_order(tmp_path: Path, monkeypa
     make_database(db)
     loaded = load_atoms(db)
     assert [atom.atom_id for atom in loaded] == ["a1", "a2", "a3"]
-    # The database stores the product's content-hash id; the caller gets the corpus id
-    # the gold annotations use.
+    # The database stores the product's content-hash id; the caller gets the corpus id the gold annotations use.
     assert [atom.source_id for atom in loaded] == ["src_red", "src_blue", "src_red"]
     assert loaded[2].text == "the key hangs on the red door"