Score recall over the whole suite and time only a stratified sample

The 300-probe sample this harness scored by modality read the screenshot
modality 0.818 where all 320 of its answerable queries read 0.659, because
striding a file-ordered list and truncating at 300 reshuffles the modality
mix. Recall is now scored over every answerable query (2,928) and latency
over a stratified sample, which is what latency needs and what recall does
not. The separate conversation sub-pass is gone: conversation is inside the
suite, and by_category already carries its six LongMemEval types.

bench/leg_check.py

@@ -2,11 +2,14 @@
 
 The product fuses three legs with reciprocal rank fusion: a sparse page, a dense vector leg, and a
 picture leg that only speaks up when the words come back thin. Each one costs latency and a model
-snapshot, and the suite is the only honest referee. This harness turns each leg on and off over the
-same probes and prints recall, first-hit quality, latency, and per-modality recall in one table, so
-a default like `dense_embeddings_enabled=False` is a measurement rather than a preference.
+snapshot, and the suite is the only honest referee. This harness turns each leg on and off over
+**every answerable query in the suite** and prints recall, first-hit quality, latency, and
+per-modality recall in one table, so a default like `dense_embeddings_enabled=False` is a
+measurement rather than a preference. Recall is scored whole because a 300-query sample reads a
+modality a few points off; latency is timed on a stratified sample because that is what a sample
+is for.
 
-Run: python -m bench.leg_check [--probes 300]
+Run: python -m bench.leg_check [--probes 300]   # --probes sizes the timing sample only
 Writes: bench/reports/LEGS.md, bench/reports/leg_check.json
 """
 
@@ -40,9 +43,6 @@ ENCODE_SAMPLE = 200
 # The page the product asks for by default; measure() checks it against Settings, so "as shipped" cannot rot into a hypothesis.
 SHIPPED_TOP_K = 24
 
-# The modality whose questions carry a question type; measured whole, because 15 probes of 150 cannot separate six categories.
-MEMORY_MODALITY = "conversation"
-
 # (label, atoms asked, dense leg, picture leg); the picture leg answers only on a thin page, which is the product's rule.
 CONFIGS: list[dict[str, Any]] = [
     {"label": "words only", "top_k": 10, "dense": False, "visual": False},
@@ -232,16 +232,18 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
             f"SHIPPED_TOP_K={SHIPPED_TOP_K}; update the rows before trusting the 'as shipped' label"
         )
 
-    chosen = scale_probes(load_queries(), probes)
-    answerable = [query for query in chosen if query.gold]
+    every = load_queries()
+    answerable = [query for query in every if query.gold]
     if not answerable:
         raise SystemExit("the frozen suite carries no answerable query; nothing to score")
-    print(f"{len(answerable)} answerable probes", flush=True)
+    timed = {query.query_id for query in scale_probes(every, probes) if query.gold}
+    if not timed:
+        raise SystemExit(
+            f"--probes {probes} picked no query to time: the sample strides the suite by modality, "
+            "so a small count can skip one and the latency row would be a single modality"
+        )
+    print(f"{len(answerable)} answerable queries scored, {len(timed)} of them timed", flush=True)
     times = source_times()
-    memory = [query for query in load_queries() if query.modality == MEMORY_MODALITY and query.gold]
-    if not memory:
-        raise SystemExit(f"the frozen suite carries no answerable {MEMORY_MODALITY} query; the memory tables would be empty")
-    print(f"{len(memory)} answerable {MEMORY_MODALITY} queries, measured whole beside the probes", flush=True)
 
     vectors = _dense_vectors(home / "data" / "ely_eye.sqlite")
     source_schema = _schema(home)
@@ -287,13 +289,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
             for query, ranking in zip(answerable, rankings, strict=True)
         ]
         summary = aggregate(scores)
-        timing = latency(timings)
-        memory_rankings, _, _ = rank_all(service, ids, memory, int(config["top_k"]))
-        memory_scores = [
-            score_query(query.query_id, ranking, set(query.gold), query.holder)
-            for query, ranking in zip(memory, memory_rankings, strict=True)
-        ]
-        memory_summary = aggregate(memory_scores)
+        timing = latency([t for query, t in zip(answerable, timings, strict=True) if query.query_id in timed])
         arms.append(
             {
                 "label": config["label"],
@@ -308,14 +304,6 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
                 "p95_seconds": timing["p95"],
                 "median_distinct_sources": statistics.median(distinct),
                 **_groups(answerable, scores, rankings, times),
-                "memory": {
-                    "queries": len(memory),
-                    "recall_at_10": memory_summary["recall_at_10"],
-                    "mrr": memory_summary["mrr"],
-                    "hit_at_10": memory_summary["hit_at_10"],
-                    "wilson_lower": memory_summary["hit_at_10_wilson_lower"],
-                    **_groups(memory, memory_scores, memory_rankings, times),
-                },
             }
         )
         print(
@@ -333,9 +321,8 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
         "dense_vectors": vectors,
         "picture_index": picture,
         "encode_rate": rate,
-        "probes": len(answerable),
-        "memory_queries": len(memory),
-        "memory_modality": MEMORY_MODALITY,
+        "queries": len(answerable),
+        "timed_queries": len(timed),
         "arms": arms,
         "machine": machine_facts(),
     }

bench/leg_report.py

@@ -32,9 +32,9 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
         )
     counts = report["arms"][0]["modality_counts"]
 
-    def memory_table(title: str, field: str, sizes_field: str) -> list[str]:
-        """A table over the question types the memory set labels; unlabelled sets stay out of it."""
-        rows = [arm["memory"] for arm in report["arms"]]
+    def category_table(title: str, field: str, sizes_field: str) -> list[str]:
+        """A table over the question types the set records; unlabelled sets stay out of it."""
+        rows = report["arms"]
         names = sorted({name for row in rows for name in row[field] if name != "unlabelled"})
         if not names:
             return []
@@ -79,11 +79,12 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
                     "sits outside the measured latency, and the frozen baseline itself holds no "
                     "vectors, so a dense arm there would rank an empty index."
                 ),
-                f"- {report['probes']} answerable queries, each asked as the holder the frozen set assigns it",
                 (
-                    f"- memory subset: every answerable {report['memory_modality']} query "
-                    f"({report['memory_queries']}) measured whole, because a 15-probe sample cannot "
-                    "separate six question types; the two tables further down use that subset, not the sample"
+                    f"- {report['queries']:,} answerable queries scored whole, each asked as the "
+                    f"holder the frozen set assigns it; latency is timed on a stratified sample of "
+                    f"{report['timed_queries']}, because a sample sized for timing is not sized for a "
+                    "modality's recall — the 300-probe sample this harness used before read the "
+                    "screenshot modality 0.818 where all 320 of its answerable queries read 0.659"
                 ),
                 *(
                     []
@@ -121,9 +122,9 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
                 "",
                 f"_Query counts per modality: {', '.join(f'{name} {count}' for name, count in counts.items())}._",
                 "",
-                *memory_table("recall@10 by question type, every memory query measured", "by_category", "category_counts"),
+                *category_table("recall@10 by question type", "by_category", "category_counts"),
                 "",
-                *memory_table(
+                *category_table(
                     "the newest of the gold memories came back first",
                     "newest_gold_first_by_category",
                     "newest_gold_first_counts",

bench/tests/test_leg_check.py

@@ -33,18 +33,10 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
         "by_modality": {"code": 0.953, "pdf": 0.849},
         "modality_counts": {"code": 128, "pdf": 86},
         "newest_gold_first": 0.44,
-        "memory": {
-            "queries": 150,
-            "recall_at_10": 0.965,
-            "mrr": 0.875,
-            "hit_at_10": 0.98,
-            "wilson_lower": 0.921,
-            "by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947},
-            "category_counts": {"knowledge-update": 25, "temporal-reasoning": 25},
-            "newest_gold_first": 0.4,
-            "newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526},
-            "newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21},
-        },
+        "by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947},
+        "category_counts": {"knowledge-update": 25, "temporal-reasoning": 25},
+        "newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526},
+        "newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21},
     }
     report: dict[str, Any] = {
         "commit": "abcdef123456",
@@ -55,9 +47,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
         "dense_vectors": 153857,
         "picture_index": {"path": "bench/runs/leg_check_index", "atoms": 53668, "dim": 512, "bytes": 109912192},
         "encode_rate": {"pictures": 200, "seconds": 10.0, "per_second": 20.0},
-        "probes": 300,
-        "memory_queries": 150,
-        "memory_modality": "conversation",
+        "queries": 2928,
+        "timed_queries": 300,
         "arms": [arm],
         "machine": {
             "system": "Linux",
@@ -83,8 +74,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
         "knowledge-update" in text and "1.000" in text and "0.40 (19)" in text
     ), "the two question types the contract names need their own recall and their own n"
     assert (
-        "every answerable conversation query (150) measured whole" in text
-    ), "a 15-probe sample cannot separate six question types, and the report must say which set the table used"
+        "2,928 answerable queries scored whole" in text and "stratified sample of 300" in text
+    ), "recall is scored over the whole suite and only latency is sampled, and the report must say so"
     assert "| 0.44 |" in text, "the probe table needs its own newest-first column"
     assert (
         "arrives at v7 and the product migrates it when it opens, so every arm ran on v8" in text