Name the schema a leg report actually measured
LEGS.md said "schema v9" while every arm ran on v10. The harness read PRAGMA user_version straight after copying the store, before the product had opened it, so it printed the version the copy arrived with. The copy is migrated on first open, and that is the schema the retrieval service ran on. The version is now read after the arms, and the report line says both: the copy arrives at v9, the product migrates it when it opens, every arm ran on v10. A test pins the sentence so the two versions cannot collapse back into one number. Gates: bench/tests 68 passed; ruff clean; mypy clean on the three files.
This commit is contained in:
1 parent
dc9cfb5a30
commit
a52c10499b
3 files changed
+15
-4
No files matched your search
+6
-3
@@ -244,6 +244,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
|||||||
print(f"{len(memory)} answerable {MEMORY_MODALITY} queries, measured whole beside the probes", flush=True)
|
print(f"{len(memory)} answerable {MEMORY_MODALITY} queries, measured whole beside the probes", flush=True)
|
||||||
|
|
||||||
vectors = _dense_vectors(home / "data" / "ely_eye.sqlite")
|
vectors = _dense_vectors(home / "data" / "ely_eye.sqlite")
|
||||||
|
source_schema = _schema(home)
|
||||||
if vectors == 0 and any(config["dense"] for config in CONFIGS):
|
if vectors == 0 and any(config["dense"] for config in CONFIGS):
|
||||||
raise SystemExit(
|
raise SystemExit(
|
||||||
f"{home}: the store carries no dense vectors, so a dense arm here would rank an empty "
|
f"{home}: the store carries no dense vectors, so a dense arm here would rank an empty "
|
||||||
@@ -254,7 +255,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
|||||||
arms: list[dict[str, Any]] = []
|
arms: list[dict[str, Any]] = []
|
||||||
picture: dict[str, Any] | None = None
|
picture: dict[str, Any] | None = None
|
||||||
rate: dict[str, Any] | None = None
|
rate: dict[str, Any] | None = None
|
||||||
schema = 0
|
copy_home = home
|
||||||
for config in CONFIGS:
|
for config in CONFIGS:
|
||||||
print(f" {config['label']}: dense={config['dense']} visual={config['visual']}", flush=True)
|
print(f" {config['label']}: dense={config['dense']} visual={config['visual']}", flush=True)
|
||||||
copy_home = prepare(
|
copy_home = prepare(
|
||||||
@@ -267,7 +268,6 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
|||||||
clip_model=models["clip"],
|
clip_model=models["clip"],
|
||||||
embedding_model=models["embedding"],
|
embedding_model=models["embedding"],
|
||||||
)
|
)
|
||||||
schema = _schema(copy_home)
|
|
||||||
if config["visual"]:
|
if config["visual"]:
|
||||||
picture = picture_facts(clip_index)
|
picture = picture_facts(clip_index)
|
||||||
if rate is None:
|
if rate is None:
|
||||||
@@ -323,9 +323,12 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
|||||||
f"p50 {timing['p50'] * 1000:.1f}ms",
|
f"p50 {timing['p50'] * 1000:.1f}ms",
|
||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
|
# Read the version after the arms: the copy arrives at the source's schema and the product
|
||||||
|
# migrates it the first time it opens, so the version before the run is not the one measured.
|
||||||
return {
|
return {
|
||||||
"commit": git_commit(),
|
"commit": git_commit(),
|
||||||
"schema_version": schema,
|
"schema_version": _schema(copy_home),
|
||||||
|
"source_schema_version": source_schema,
|
||||||
"store": str(scratch / "data" / "ely_eye.sqlite"),
|
"store": str(scratch / "data" / "ely_eye.sqlite"),
|
||||||
"source_store": str(home),
|
"source_store": str(home),
|
||||||
"dense_vectors": vectors,
|
"dense_vectors": vectors,
|
||||||
|
|||||||
+5
-1
@@ -68,7 +68,11 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
|
|||||||
[
|
[
|
||||||
"# LEGS: what each retrieval leg is worth",
|
"# LEGS: what each retrieval leg is worth",
|
||||||
"",
|
"",
|
||||||
f"- commit: `{report['commit']}`, schema v{report['schema_version']}",
|
(
|
||||||
|
f"- commit: `{report['commit']}`, schema v{report['schema_version']} — the copy "
|
||||||
|
f"arrives at v{report['source_schema_version']} and the product migrates it when "
|
||||||
|
f"it opens, so every arm ran on v{report['schema_version']}"
|
||||||
|
),
|
||||||
(
|
(
|
||||||
f"- store: {report['store']} — a copy of {report['source_store']}, which carries "
|
f"- store: {report['store']} — a copy of {report['source_store']}, which carries "
|
||||||
f"{report['dense_vectors']:,} dense vectors built by bench/dense_scale. The build "
|
f"{report['dense_vectors']:,} dense vectors built by bench/dense_scale. The build "
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
|
|||||||
report: dict[str, Any] = {
|
report: dict[str, Any] = {
|
||||||
"commit": "abcdef123456",
|
"commit": "abcdef123456",
|
||||||
"schema_version": 8,
|
"schema_version": 8,
|
||||||
|
"source_schema_version": 7,
|
||||||
"store": "bench/runs/leg_check/data/ely_eye.sqlite",
|
"store": "bench/runs/leg_check/data/ely_eye.sqlite",
|
||||||
"source_store": "bench/runs/dense_scale",
|
"source_store": "bench/runs/dense_scale",
|
||||||
"dense_vectors": 153857,
|
"dense_vectors": 153857,
|
||||||
@@ -85,5 +86,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
|
|||||||
"every answerable conversation query (150) measured whole" in text
|
"every answerable conversation query (150) measured whole" in text
|
||||||
), "a 15-probe sample cannot separate six question types, and the report must say which set the table used"
|
), "a 15-probe sample cannot separate six question types, and the report must say which set the table used"
|
||||||
assert "| 0.44 |" in text, "the probe table needs its own newest-first column"
|
assert "| 0.44 |" in text, "the probe table needs its own newest-first column"
|
||||||
|
assert (
|
||||||
|
"arrives at v7 and the product migrates it when it opens, so every arm ran on v8" in text
|
||||||
|
), "the schema named in the report has to be the one the product ran on, not the one the copy arrived with"
|
||||||
assert "unlabelled" not in text, "an unlabelled bucket would hide which set a number came from"
|
assert "unlabelled" not in text, "an unlabelled bucket would hide which set a number came from"
|
||||||
assert "abcdef123456" in text and "load_average" not in text
|
assert "abcdef123456" in text and "load_average" not in text
|
||||||
Reference in new issue
Block a user