Stop the checklist from quoting numbers no artifact carries

A review of the p2-1 record found seven places where the prose had drifted
away from the reports it cites. None were product defects; all were record
keeping, and record keeping is what a memory system is judged on.

The quantization row mis-transcribed nine latencies, and the two worst made
dimension truncation look twice as fast as the full scan when the report
measures six percent. The extrapolation table scaled latency by atoms and
bytes by vectors under one heading that named neither switch. The picture
index stats existed only in prose, so the harness now reads the index shape
off the artifact and times the encoder on a sample, which reproduces the
hand-timed build within seven percent. The dense arms named a store that
holds no vectors, so the report names the store the vectors came from and
refuses to run where a dense leg would rank an empty index.

The frozen baseline was migrated in place by the product CLI while the
picture index was built, so its pinned mtime, digest and schema version are
re-pinned and the step that moved it is named. The sqlite_vec import added
two mypy errors; the fix is the missing stub declaration, not an inline
ignore, and the count is back to the recorded baseline of twenty-four.

bench/dense_scale.py

@@ -192,13 +192,16 @@ def measure(home: Path, scratch: Path, probes: int, model_dir: Path) -> dict[str
         "bytes_method": bytes_method,
         "extrapolation": {
             "to": EXTRAPOLATE_TO,
+            "basis": encoded,
             "assumption": (
                 "sqlite-vec 0.1.9 scans every vector, so latency and bytes grow linearly with the "
-                "count; the model, the machine and the thread budget stay the same"
+                "vector count; every row here is scaled by "
+                f"{EXTRAPOLATE_TO} / {encoded} vectors, and the model, the machine and the thread "
+                "budget stay the same"
             ),
-            "p50_seconds": timing["p50"] * EXTRAPOLATE_TO / atoms,
-            "p95_seconds": timing["p95"] * EXTRAPOLATE_TO / atoms,
-            "write_p50_seconds": writes["p50"] * EXTRAPOLATE_TO / atoms,
+            "p50_seconds": timing["p50"] * EXTRAPOLATE_TO / encoded,
+            "p95_seconds": timing["p95"] * EXTRAPOLATE_TO / encoded,
+            "write_p50_seconds": writes["p50"] * EXTRAPOLATE_TO / encoded,
             "gibibytes": per_vector * EXTRAPOLATE_TO / 1024**3,
             "build_seconds": EXTRAPOLATE_TO / texts_per_second if texts_per_second else None,
         },
@@ -254,12 +257,12 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
                     f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"
                 ),
                 "",
-                "## One million memories on this machine",
+                "## One million vectors on this machine",
                 "",
                 f"- p50 {future['p50_seconds'] * 1000:.0f}ms · p95 {future['p95_seconds'] * 1000:.0f}ms",
                 f"- vector table: {future['gibibytes']:.1f} GiB",
                 f"- first build: {future['build_seconds'] / 60:.1f} minutes",
-                f"- writing one memory: {future['write_p50_seconds'] * 1000:.0f}ms — a vec0 delete by id scans the table, so this grows with the count too",
+                f"- writing one vector: {future['write_p50_seconds'] * 1000:.0f}ms — a vec0 delete by id scans the table, so this grows with the count too",
                 f"- assumption: {future['assumption']}",
                 "",
                 (

bench/leg_check.py

@@ -34,13 +34,12 @@ DEFAULT_OBJECTS = Path("bench/runs/baseline/objects")
 DEFAULT_CLIP_INDEX = Path("bench/runs/leg_check_index")
 DEFAULT_CLIP_MODEL = Path("bench/models/chinese-clip-vit-base-patch16")
 DEFAULT_EMBEDDING_MODEL = Path("bench/models/paraphrase-multilingual-MiniLM-L12-v2")
+ENCODE_SAMPLE = 200
 
-# The page size the product asks for by default. measure() checks it against the product's own
-# setting, so a config change here cannot quietly turn the "as shipped" row into a hypothetical.
+# The page the product asks for by default; measure() checks it against Settings, so "as shipped" cannot rot into a hypothesis.
 SHIPPED_TOP_K = 24
 
-# (label, atoms asked, dense leg, picture leg). The picture leg routes itself: it answers only when
-# the sparse page comes back thin, which is the product's rule, not this harness's.
+# (label, atoms asked, dense leg, picture leg); the picture leg answers only on a thin page, which is the product's rule.
 CONFIGS: list[dict[str, Any]] = [
     {"label": "words only", "top_k": 10, "dense": False, "visual": False},
     {"label": "words only (the configured 24)", "top_k": SHIPPED_TOP_K, "dense": False, "visual": False},
@@ -92,6 +91,74 @@ def prepare(
     return copy_home
 
 
+def _dense_vectors(store: Path) -> int:
+    """Vectors the source store already carries, read without opening it for write."""
+    import sqlite3
+
+    import sqlite_vec
+    from ely_eye import vector_index
+
+    if vector_index.TABLE != "vec_atoms":
+        raise SystemExit(f"the dense table is now {vector_index.TABLE}; update this harness's count")
+    if not store.is_file():
+        raise SystemExit(f"{store}: no product database")
+    with sqlite3.connect(f"file:{store}?mode=ro", uri=True) as connection:
+        connection.enable_load_extension(True)
+        sqlite_vec.load(connection)
+        connection.enable_load_extension(False)
+        if not connection.execute("SELECT 1 FROM sqlite_master WHERE name = 'vec_atoms'").fetchone():
+            return 0
+        return int(connection.execute("SELECT count(*) FROM vec_atoms").fetchone()[0])
+
+
+def picture_facts(clip_index: Path) -> dict[str, Any]:
+    """What the picture index really holds, read off the artifact instead of remembered in prose."""
+    import numpy as np
+
+    vectors_path = clip_index / "clip_image_index.npy"
+    atoms_path = clip_index / "clip_image_index_atoms.json"
+    vectors = np.load(vectors_path)
+    with atoms_path.open(encoding="utf-8") as handle:
+        try:
+            atoms = json.load(handle)
+        except json.JSONDecodeError as error:
+            raise ValueError(f"{atoms_path}: the picture index's atom list is unreadable ({error})") from error
+    if len(atoms) != int(vectors.shape[0]):
+        raise SystemExit(
+            f"{clip_index}: the picture index disagrees with itself — {len(atoms)} atom ids for "
+            f"{vectors.shape[0]} vectors"
+        )
+    return {
+        "path": str(clip_index),
+        "atoms": len(atoms),
+        "dim": int(vectors.shape[-1]),
+        "bytes": int(vectors_path.stat().st_size),
+    }
+
+
+def encode_rate(clip_model: Path, objects: Path, count: int) -> dict[str, Any]:
+    """Pictures per second the encoder manages, measured on a sample instead of remembered."""
+    import time
+
+    from ely_eye.clip_encoder import ClipEncoder
+    from PIL import Image
+
+    suffixes = {".png", ".jpg", ".jpeg"}
+    sample = sorted(p for p in objects.rglob("*") if p.suffix.lower() in suffixes)[:count]
+    if len(sample) < count:
+        raise SystemExit(f"{objects}: only {len(sample)} pictures to time the encoder on")
+    encoder = ClipEncoder(clip_model)
+    began = time.perf_counter()
+    rows = 0
+    for start in range(0, len(sample), 8):
+        batch = [Image.open(path).convert("RGB") for path in sample[start : start + 8]]
+        rows += len(encoder.encode_images(batch))
+    seconds = time.perf_counter() - began
+    if rows != len(sample):
+        raise SystemExit(f"the encoder returned {rows} vectors for {len(sample)} pictures")
+    return {"pictures": rows, "seconds": seconds, "per_second": rows / seconds}
+
+
 def _schema(home: Path) -> int:
     import sqlite3
 
@@ -141,7 +208,17 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
         raise SystemExit("the frozen suite carries no answerable query; nothing to score")
     print(f"{len(answerable)} answerable probes", flush=True)
 
+    vectors = _dense_vectors(home / "data" / "ely_eye.sqlite")
+    if vectors == 0 and any(config["dense"] for config in CONFIGS):
+        raise SystemExit(
+            f"{home}: the store carries no dense vectors, so a dense arm here would rank an empty "
+            "index and the 24-minute build would hide outside the measured latency. Build them with "
+            "python -m bench.dense_scale, or point --home at a store that already has them."
+        )
+
     arms: list[dict[str, Any]] = []
+    picture: dict[str, Any] | None = None
+    rate: dict[str, Any] | None = None
     schema = 0
     for config in CONFIGS:
         print(f"  {config['label']}: dense={config['dense']} visual={config['visual']}", flush=True)
@@ -156,6 +233,10 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
             embedding_model=models["embedding"],
         )
         schema = _schema(copy_home)
+        if config["visual"]:
+            picture = picture_facts(clip_index)
+            if rate is None:
+                rate = encode_rate(models["clip"], objects, ENCODE_SAMPLE)
         settings = Settings(
             home=copy_home,
             dense_embeddings_enabled=config["dense"],
@@ -203,6 +284,10 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
         "commit": git_commit(),
         "schema_version": schema,
         "store": str(scratch / "data" / "ely_eye.sqlite"),
+        "source_store": str(home),
+        "dense_vectors": vectors,
+        "picture_index": picture,
+        "encode_rate": rate,
         "probes": len(answerable),
         "arms": arms,
         "machine": machine_facts(),
@@ -247,8 +332,32 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
                 "# LEGS: what each retrieval leg is worth",
                 "",
                 f"- commit: `{report['commit']}`, schema v{report['schema_version']}",
-                f"- store: {report['store']} (a copy of {DEFAULT_HOME}; the frozen baseline is never opened)",
+                (
+                    f"- store: {report['store']} — a copy of {report['source_store']}, which carries "
+                    f"{report['dense_vectors']:,} dense vectors built by bench/dense_scale. The build "
+                    "sits outside the measured latency, and the frozen baseline itself holds no "
+                    "vectors, so a dense arm there would rank an empty index."
+                ),
                 f"- {report['probes']} answerable queries, each asked as the holder the frozen set assigns it",
+                *(
+                    []
+                    if report.get("picture_index") is None
+                    else [
+                        (
+                            f"- picture index: {report['picture_index']['atoms']:,} atoms at "
+                            f"{report['picture_index']['dim']} dimensions, "
+                            f"{report['picture_index']['bytes']:,} bytes, read from "
+                            f"{report['picture_index']['path']}"
+                        ),
+                        (
+                            f"- encoder speed: {report['encode_rate']['per_second']:.1f} pictures/s "
+                            f"measured on {report['encode_rate']['pictures']} pictures, so the "
+                            f"{report['picture_index']['atoms']:,} pictures in that index cost about "
+                            f"{report['picture_index']['atoms'] / report['encode_rate']['per_second'] / 60:.1f} "
+                            "minutes to encode"
+                        ),
+                    ]
+                ),
                 (
                     f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, "
                     f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"

bench/tests/test_leg_check.py

@@ -37,6 +37,10 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
         "commit": "abcdef123456",
         "schema_version": 8,
         "store": "bench/runs/leg_check/data/ely_eye.sqlite",
+        "source_store": "bench/runs/dense_scale",
+        "dense_vectors": 153857,
+        "picture_index": {"path": "bench/runs/leg_check_index", "atoms": 53668, "dim": 512, "bytes": 109912192},
+        "encode_rate": {"pictures": 200, "seconds": 10.0, "per_second": 20.0},
         "probes": 300,
         "arms": [arm],
         "machine": {
@@ -53,4 +57,10 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
     assert "words only" in text
     assert "0.832" in text and "63.7" in text, "recall and latency must sit in the same row"
     assert "code" in text and "0.953" in text, "per-modality recall is where the picture legs show up"
+    assert (
+        "bench/runs/dense_scale" in text and "153,857 dense vectors" in text
+    ), "a dense arm must name the store its vectors came from, not imply the frozen baseline"
+    assert (
+        "53,668 atoms at 512 dimensions" in text and "44.7" in text
+    ), "the picture index needs its shape in the report, and its build cost derived from a measured rate"
     assert "abcdef123456" in text and "load_average" not in text