Let every harness work on a copy of the frozen store

copy_store lived in dense_scale, so the two harnesses that also needed it reached back into a
measurement module for a plumbing helper. It now sits in systems beside the product path it
protects, and bench.run passes a scratch directory to the live check: the product path migrates
whatever database it opens, and the frozen baseline is cited evidence.

bench/dense_scale.py

@@ -10,7 +10,6 @@ from __future__ import annotations
 
 import argparse
 import json
-import shutil
 import sqlite3
 import time
 from collections.abc import Sequence
@@ -22,7 +21,7 @@ import numpy as np
 from bench.query_set import load_queries, scale_probes
 from bench.report import git_commit
 from bench.run import latency
-from bench.systems import DENSE_MODEL, MODELS, machine_facts
+from bench.systems import DENSE_MODEL, MODELS, copy_store, machine_facts
 
 if TYPE_CHECKING:
     from ely_eye.db import Database
@@ -65,20 +64,6 @@ def vector_footprint(database: Any, table: str) -> tuple[int, str]:
     return int(payload), "vector bytes only, this sqlite3 has no dbstat"
 
 
-def copy_store(home: Path, scratch: Path) -> tuple[Path, float]:
-    """Work on a copy: the baseline store is frozen evidence and must not be migrated in place."""
-    source = home / "data" / "ely_eye.sqlite"
-    if not source.is_file():
-        raise SystemExit(f"{source}: no product database, build it with python -m bench.ingest_baseline")
-    data = scratch / "data"
-    data.mkdir(parents=True, exist_ok=True)
-    began = time.perf_counter()
-    for suffix in ("", "-wal", "-shm"):
-        if (path := Path(str(source) + suffix)).is_file():
-            shutil.copyfile(path, data / (source.name + suffix))
-    return scratch, time.perf_counter() - began
-
-
 def stored_vectors(database: Database) -> tuple[list[str], np.ndarray]:
     """Every vector the engine holds, for a scan it can be checked against."""
     with database.connect() as connection:

bench/run.py

@@ -73,6 +73,7 @@ def live_check(
     fast_rankings: dict[str, list[str]],
     top_k: int = 10,
     live_queries: int = LIVE_QUERIES,
+    scratch: Path | None = None,
 ) -> dict[str, Any]:
     """Score the unmodified RetrievalService on its own, as its holder would ask it.
 
@@ -85,7 +86,11 @@ def live_check(
     if not chosen:
         raise SystemExit("live check needs answerable queries")
     rankings, distinct, timings = live_product_rankings(
-        DB_HOME, [query.text for query in chosen], top_k, as_holders=[query.holder for query in chosen]
+        DB_HOME,
+        [query.text for query in chosen],
+        top_k,
+        as_holders=[query.holder for query in chosen],
+        scratch=scratch,
     )
     overlaps = [
         len(set(fast_rankings[query.query_id]) & set(ranking)) / top_k
@@ -229,6 +234,12 @@ def main() -> int:
     parser.add_argument("--limit", type=int, default=None, help="score at most this many queries per modality")
     parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path")
     parser.add_argument("--live-queries", type=int, default=LIVE_QUERIES)
+    parser.add_argument(
+        "--live-scratch",
+        type=Path,
+        default=ROOT / "runs" / "live",
+        help="copy the store here before the product path opens it, so the frozen baseline stays frozen",
+    )
     parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve")
     parser.add_argument(
         "--remeasure-latency",
@@ -325,7 +336,9 @@ def main() -> int:
     # A crash in the live or scale phase should not cost every finished arm.
     (DB_HOME / "partial_report.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8")
     if not args.no_live:
-        report["live_product"] = live_check(queries, fast_rankings, live_queries=args.live_queries)
+        report["live_product"] = live_check(
+            queries, fast_rankings, live_queries=args.live_queries, scratch=args.live_scratch
+        )
     if not args.no_scale:
         report["scale"] = scale_curve(queries, atoms, [arm for arm in arms_requested if arm in ("bm25_only", "dense_only")])
 

bench/scale_check.py

@@ -14,11 +14,10 @@ import time
 from pathlib import Path
 from typing import Any
 
-from bench.dense_scale import copy_store
 from bench.query_set import load_queries, scale_probes
 from bench.report import git_commit
 from bench.run import latency
-from bench.systems import machine_facts
+from bench.systems import copy_store, machine_facts
 
 DEFAULT_HOME = Path("bench/runs/baseline")
 DEFAULT_SCRATCH = Path("bench/runs/scale")

bench/systems.py

@@ -13,6 +13,7 @@ import json
 import os
 import platform
 import re
+import shutil
 import sqlite3
 import time
 from collections.abc import Sequence
@@ -445,11 +446,26 @@ def _corpus_ids(database: Database) -> dict[str, str]:
     return ids
 
 
+def copy_store(home: Path, scratch: Path) -> tuple[Path, float]:
+    """Work on a copy: the baseline store is frozen evidence and must not be migrated in place."""
+    source = home / "data" / "ely_eye.sqlite"
+    if not source.is_file():
+        raise SystemExit(f"{source}: no product database, build it with python -m bench.ingest_baseline")
+    data = scratch / "data"
+    data.mkdir(parents=True, exist_ok=True)
+    began = time.perf_counter()
+    for suffix in ("", "-wal", "-shm"):
+        if (path := Path(str(source) + suffix)).is_file():
+            shutil.copyfile(path, data / (source.name + suffix))
+    return scratch, time.perf_counter() - began
+
+
 def live_product_rankings(
     db_home: Path,
     queries: Sequence[str],
     top_k: int = 10,
     as_holders: Sequence[str | None] | None = None,
+    scratch: Path | None = None,
 ) -> tuple[list[list[str]], list[int], list[float]]:
     """Rankings from the unmodified RetrievalService, with per-query wall time.
 
@@ -460,6 +476,9 @@ def live_product_rankings(
     The second list counts how many distinct sources the product's top-k atoms came from: the
     product ranks atoms and never diversifies, so ten slots can be filled by fragments of one
     document, and that is invisible once the hits are collapsed.
+
+    `scratch` copies the store before opening it. The product path migrates whatever it opens, and
+    the baseline store is cited evidence, so a benchmark that measures the product must not edit it.
     """
     from ely_eye.config import Settings
     from ely_eye.db import Database
@@ -468,6 +487,8 @@ def live_product_rankings(
 
     if as_holders is not None and len(as_holders) != len(queries):
         raise SystemExit(f"{len(as_holders)} holders for {len(queries)} queries")
+    if scratch is not None:
+        db_home, _ = copy_store(db_home, scratch)
     settings = Settings(home=db_home)
     database = Database(settings)
     service = RetrievalService(settings=settings, database=database)

bench/term_sweep.py

@@ -16,12 +16,11 @@ import statistics
 from pathlib import Path
 from typing import Any
 
-from bench.dense_scale import copy_store
 from bench.metrics import aggregate, score_query
 from bench.query_set import load_queries, scale_probes
 from bench.report import git_commit
 from bench.run import latency
-from bench.systems import live_product_rankings, machine_facts
+from bench.systems import copy_store, live_product_rankings, machine_facts
 
 FRACTIONS = [0.20, 0.10, 0.05, 0.02, 0.01, 0.005]
 TOP_K = 10