diff --git a/bench/leg_check.py b/bench/leg_check.py index fb6d27c..a1e9842 100644 --- a/bench/leg_check.py +++ b/bench/leg_check.py @@ -244,6 +244,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: print(f"{len(memory)} answerable {MEMORY_MODALITY} queries, measured whole beside the probes", flush=True) vectors = _dense_vectors(home / "data" / "ely_eye.sqlite") + source_schema = _schema(home) if vectors == 0 and any(config["dense"] for config in CONFIGS): raise SystemExit( f"{home}: the store carries no dense vectors, so a dense arm here would rank an empty " @@ -254,7 +255,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: arms: list[dict[str, Any]] = [] picture: dict[str, Any] | None = None rate: dict[str, Any] | None = None - schema = 0 + copy_home = home for config in CONFIGS: print(f" {config['label']}: dense={config['dense']} visual={config['visual']}", flush=True) copy_home = prepare( @@ -267,7 +268,6 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: clip_model=models["clip"], embedding_model=models["embedding"], ) - schema = _schema(copy_home) if config["visual"]: picture = picture_facts(clip_index) if rate is None: @@ -323,9 +323,12 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: f"p50 {timing['p50'] * 1000:.1f}ms", flush=True, ) + # Read the version after the arms: the copy arrives at the source's schema and the product + # migrates it the first time it opens, so the version before the run is not the one measured. return { "commit": git_commit(), - "schema_version": schema, + "schema_version": _schema(copy_home), + "source_schema_version": source_schema, "store": str(scratch / "data" / "ely_eye.sqlite"), "source_store": str(home), "dense_vectors": vectors, diff --git a/bench/leg_report.py b/bench/leg_report.py index 5309615..40596d2 100644 --- a/bench/leg_report.py +++ b/bench/leg_report.py @@ -68,7 +68,11 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: [ "# LEGS: what each retrieval leg is worth", "", - f"- commit: `{report['commit']}`, schema v{report['schema_version']}", + ( + f"- commit: `{report['commit']}`, schema v{report['schema_version']} — the copy " + f"arrives at v{report['source_schema_version']} and the product migrates it when " + f"it opens, so every arm ran on v{report['schema_version']}" + ), ( f"- store: {report['store']} — a copy of {report['source_store']}, which carries " f"{report['dense_vectors']:,} dense vectors built by bench/dense_scale. The build " diff --git a/bench/tests/test_leg_check.py b/bench/tests/test_leg_check.py index 54acc71..2146582 100644 --- a/bench/tests/test_leg_check.py +++ b/bench/tests/test_leg_check.py @@ -49,6 +49,7 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None: report: dict[str, Any] = { "commit": "abcdef123456", "schema_version": 8, + "source_schema_version": 7, "store": "bench/runs/leg_check/data/ely_eye.sqlite", "source_store": "bench/runs/dense_scale", "dense_vectors": 153857, @@ -85,5 +86,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None: "every answerable conversation query (150) measured whole" in text ), "a 15-probe sample cannot separate six question types, and the report must say which set the table used" assert "| 0.44 |" in text, "the probe table needs its own newest-first column" + assert ( + "arrives at v7 and the product migrates it when it opens, so every arm ran on v8" in text + ), "the schema named in the report has to be the one the product ran on, not the one the copy arrived with" assert "unlabelled" not in text, "an unlabelled bucket would hide which set a number came from" assert "abcdef123456" in text and "load_average" not in text