Name the schema a leg report actually measured

LEGS.md said "schema v9" while every arm ran on v10. The harness read
PRAGMA user_version straight after copying the store, before the product
had opened it, so it printed the version the copy arrived with. The copy is
migrated on first open, and that is the schema the retrieval service ran on.

The version is now read after the arms, and the report line says both: the
copy arrives at v9, the product migrates it when it opens, every arm ran on
v10. A test pins the sentence so the two versions cannot collapse back into
one number.

Gates: bench/tests 68 passed; ruff clean; mypy clean on the three files.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-09 07:32:45 -04:00
1 parent dc9cfb5a30
commit a52c10499b
3 files changed
+15 -4

No files matched your search

+6 -3
View File
@@ -244,6 +244,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
print(f"{len(memory)} answerable {MEMORY_MODALITY} queries, measured whole beside the probes", flush=True)
vectors = _dense_vectors(home / "data" / "ely_eye.sqlite")
source_schema = _schema(home)
if vectors == 0 and any(config["dense"] for config in CONFIGS):
raise SystemExit(
f"{home}: the store carries no dense vectors, so a dense arm here would rank an empty "
@@ -254,7 +255,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
arms: list[dict[str, Any]] = []
picture: dict[str, Any] | None = None
rate: dict[str, Any] | None = None
schema = 0
copy_home = home
for config in CONFIGS:
print(f" {config['label']}: dense={config['dense']} visual={config['visual']}", flush=True)
copy_home = prepare(
@@ -267,7 +268,6 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
clip_model=models["clip"],
embedding_model=models["embedding"],
)
schema = _schema(copy_home)
if config["visual"]:
picture = picture_facts(clip_index)
if rate is None:
@@ -323,9 +323,12 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
f"p50 {timing['p50'] * 1000:.1f}ms",
flush=True,
)
# Read the version after the arms: the copy arrives at the source's schema and the product
# migrates it the first time it opens, so the version before the run is not the one measured.
return {
"commit": git_commit(),
"schema_version": schema,
"schema_version": _schema(copy_home),
"source_schema_version": source_schema,
"store": str(scratch / "data" / "ely_eye.sqlite"),
"source_store": str(home),
"dense_vectors": vectors,
+5 -1
View File
@@ -68,7 +68,11 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
[
"# LEGS: what each retrieval leg is worth",
"",
f"- commit: `{report['commit']}`, schema v{report['schema_version']}",
(
f"- commit: `{report['commit']}`, schema v{report['schema_version']} — the copy "
f"arrives at v{report['source_schema_version']} and the product migrates it when "
f"it opens, so every arm ran on v{report['schema_version']}"
),
(
f"- store: {report['store']} — a copy of {report['source_store']}, which carries "
f"{report['dense_vectors']:,} dense vectors built by bench/dense_scale. The build "
+4
View File
@@ -49,6 +49,7 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
report: dict[str, Any] = {
"commit": "abcdef123456",
"schema_version": 8,
"source_schema_version": 7,
"store": "bench/runs/leg_check/data/ely_eye.sqlite",
"source_store": "bench/runs/dense_scale",
"dense_vectors": 153857,
@@ -85,5 +86,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
"every answerable conversation query (150) measured whole" in text
), "a 15-probe sample cannot separate six question types, and the report must say which set the table used"
assert "| 0.44 |" in text, "the probe table needs its own newest-first column"
assert (
"arrives at v7 and the product migrates it when it opens, so every arm ran on v8" in text
), "the schema named in the report has to be the one the product ran on, not the one the copy arrived with"
assert "unlabelled" not in text, "an unlabelled bucket would hide which set a number came from"
assert "abcdef123456" in text and "load_average" not in text