From 91fbff34ee666da2a557cb1384c63cf15eb2695f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 15:11:27 -0400 Subject: [PATCH] Let every harness work on a copy of the frozen store copy_store lived in dense_scale, so the two harnesses that also needed it reached back into a measurement module for a plumbing helper. It now sits in systems beside the product path it protects, and bench.run passes a scratch directory to the live check: the product path migrates whatever database it opens, and the frozen baseline is cited evidence. --- bench/dense_scale.py | 17 +---------------- bench/run.py | 17 +++++++++++++++-- bench/scale_check.py | 3 +-- bench/systems.py | 21 +++++++++++++++++++++ bench/term_sweep.py | 3 +-- 5 files changed, 39 insertions(+), 22 deletions(-) diff --git a/bench/dense_scale.py b/bench/dense_scale.py index d43950f..e2a3f37 100644 --- a/bench/dense_scale.py +++ b/bench/dense_scale.py @@ -10,7 +10,6 @@ from __future__ import annotations import argparse import json -import shutil import sqlite3 import time from collections.abc import Sequence @@ -22,7 +21,7 @@ import numpy as np from bench.query_set import load_queries, scale_probes from bench.report import git_commit from bench.run import latency -from bench.systems import DENSE_MODEL, MODELS, machine_facts +from bench.systems import DENSE_MODEL, MODELS, copy_store, machine_facts if TYPE_CHECKING: from ely_eye.db import Database @@ -65,20 +64,6 @@ def vector_footprint(database: Any, table: str) -> tuple[int, str]: return int(payload), "vector bytes only, this sqlite3 has no dbstat" -def copy_store(home: Path, scratch: Path) -> tuple[Path, float]: - """Work on a copy: the baseline store is frozen evidence and must not be migrated in place.""" - source = home / "data" / "ely_eye.sqlite" - if not source.is_file(): - raise SystemExit(f"{source}: no product database, build it with python -m bench.ingest_baseline") - data = scratch / "data" - data.mkdir(parents=True, exist_ok=True) - began = time.perf_counter() - for suffix in ("", "-wal", "-shm"): - if (path := Path(str(source) + suffix)).is_file(): - shutil.copyfile(path, data / (source.name + suffix)) - return scratch, time.perf_counter() - began - - def stored_vectors(database: Database) -> tuple[list[str], np.ndarray]: """Every vector the engine holds, for a scan it can be checked against.""" with database.connect() as connection: diff --git a/bench/run.py b/bench/run.py index 68545cf..3c6254b 100644 --- a/bench/run.py +++ b/bench/run.py @@ -73,6 +73,7 @@ def live_check( fast_rankings: dict[str, list[str]], top_k: int = 10, live_queries: int = LIVE_QUERIES, + scratch: Path | None = None, ) -> dict[str, Any]: """Score the unmodified RetrievalService on its own, as its holder would ask it. @@ -85,7 +86,11 @@ def live_check( if not chosen: raise SystemExit("live check needs answerable queries") rankings, distinct, timings = live_product_rankings( - DB_HOME, [query.text for query in chosen], top_k, as_holders=[query.holder for query in chosen] + DB_HOME, + [query.text for query in chosen], + top_k, + as_holders=[query.holder for query in chosen], + scratch=scratch, ) overlaps = [ len(set(fast_rankings[query.query_id]) & set(ranking)) / top_k @@ -229,6 +234,12 @@ def main() -> int: parser.add_argument("--limit", type=int, default=None, help="score at most this many queries per modality") parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path") parser.add_argument("--live-queries", type=int, default=LIVE_QUERIES) + parser.add_argument( + "--live-scratch", + type=Path, + default=ROOT / "runs" / "live", + help="copy the store here before the product path opens it, so the frozen baseline stays frozen", + ) parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve") parser.add_argument( "--remeasure-latency", @@ -325,7 +336,9 @@ def main() -> int: # A crash in the live or scale phase should not cost every finished arm. (DB_HOME / "partial_report.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8") if not args.no_live: - report["live_product"] = live_check(queries, fast_rankings, live_queries=args.live_queries) + report["live_product"] = live_check( + queries, fast_rankings, live_queries=args.live_queries, scratch=args.live_scratch + ) if not args.no_scale: report["scale"] = scale_curve(queries, atoms, [arm for arm in arms_requested if arm in ("bm25_only", "dense_only")]) diff --git a/bench/scale_check.py b/bench/scale_check.py index 341f9ba..4bdb937 100644 --- a/bench/scale_check.py +++ b/bench/scale_check.py @@ -14,11 +14,10 @@ import time from pathlib import Path from typing import Any -from bench.dense_scale import copy_store from bench.query_set import load_queries, scale_probes from bench.report import git_commit from bench.run import latency -from bench.systems import machine_facts +from bench.systems import copy_store, machine_facts DEFAULT_HOME = Path("bench/runs/baseline") DEFAULT_SCRATCH = Path("bench/runs/scale") diff --git a/bench/systems.py b/bench/systems.py index 037a99f..033d0d2 100644 --- a/bench/systems.py +++ b/bench/systems.py @@ -13,6 +13,7 @@ import json import os import platform import re +import shutil import sqlite3 import time from collections.abc import Sequence @@ -445,11 +446,26 @@ def _corpus_ids(database: Database) -> dict[str, str]: return ids +def copy_store(home: Path, scratch: Path) -> tuple[Path, float]: + """Work on a copy: the baseline store is frozen evidence and must not be migrated in place.""" + source = home / "data" / "ely_eye.sqlite" + if not source.is_file(): + raise SystemExit(f"{source}: no product database, build it with python -m bench.ingest_baseline") + data = scratch / "data" + data.mkdir(parents=True, exist_ok=True) + began = time.perf_counter() + for suffix in ("", "-wal", "-shm"): + if (path := Path(str(source) + suffix)).is_file(): + shutil.copyfile(path, data / (source.name + suffix)) + return scratch, time.perf_counter() - began + + def live_product_rankings( db_home: Path, queries: Sequence[str], top_k: int = 10, as_holders: Sequence[str | None] | None = None, + scratch: Path | None = None, ) -> tuple[list[list[str]], list[int], list[float]]: """Rankings from the unmodified RetrievalService, with per-query wall time. @@ -460,6 +476,9 @@ def live_product_rankings( The second list counts how many distinct sources the product's top-k atoms came from: the product ranks atoms and never diversifies, so ten slots can be filled by fragments of one document, and that is invisible once the hits are collapsed. + + `scratch` copies the store before opening it. The product path migrates whatever it opens, and + the baseline store is cited evidence, so a benchmark that measures the product must not edit it. """ from ely_eye.config import Settings from ely_eye.db import Database @@ -468,6 +487,8 @@ def live_product_rankings( if as_holders is not None and len(as_holders) != len(queries): raise SystemExit(f"{len(as_holders)} holders for {len(queries)} queries") + if scratch is not None: + db_home, _ = copy_store(db_home, scratch) settings = Settings(home=db_home) database = Database(settings) service = RetrievalService(settings=settings, database=database) diff --git a/bench/term_sweep.py b/bench/term_sweep.py index 2a5116b..4c7b28a 100644 --- a/bench/term_sweep.py +++ b/bench/term_sweep.py @@ -16,12 +16,11 @@ import statistics from pathlib import Path from typing import Any -from bench.dense_scale import copy_store from bench.metrics import aggregate, score_query from bench.query_set import load_queries, scale_probes from bench.report import git_commit from bench.run import latency -from bench.systems import live_product_rankings, machine_facts +from bench.systems import copy_store, live_product_rankings, machine_facts FRACTIONS = [0.20, 0.10, 0.05, 0.02, 0.01, 0.005] TOP_K = 10