From e81923f62ebb88d017461bd4943def4260e8d4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Sat, 10 Oct 2026 01:59:25 -0400 Subject: [PATCH] Score recall over the whole suite and time only a stratified sample The 300-probe sample this harness scored by modality read the screenshot modality 0.818 where all 320 of its answerable queries read 0.659, because striding a file-ordered list and truncating at 300 reshuffles the modality mix. Recall is now scored over every answerable query (2,928) and latency over a stratified sample, which is what latency needs and what recall does not. The separate conversation sub-pass is gone: conversation is inside the suite, and by_category already carries its six LongMemEval types. --- bench/leg_check.py | 51 +++++++++++++---------------------- bench/leg_report.py | 19 ++++++------- bench/tests/test_leg_check.py | 25 ++++++----------- 3 files changed, 37 insertions(+), 58 deletions(-) diff --git a/bench/leg_check.py b/bench/leg_check.py index 977d19e..43ef148 100644 --- a/bench/leg_check.py +++ b/bench/leg_check.py @@ -2,11 +2,14 @@ The product fuses three legs with reciprocal rank fusion: a sparse page, a dense vector leg, and a picture leg that only speaks up when the words come back thin. Each one costs latency and a model -snapshot, and the suite is the only honest referee. This harness turns each leg on and off over the -same probes and prints recall, first-hit quality, latency, and per-modality recall in one table, so -a default like `dense_embeddings_enabled=False` is a measurement rather than a preference. +snapshot, and the suite is the only honest referee. This harness turns each leg on and off over +**every answerable query in the suite** and prints recall, first-hit quality, latency, and +per-modality recall in one table, so a default like `dense_embeddings_enabled=False` is a +measurement rather than a preference. Recall is scored whole because a 300-query sample reads a +modality a few points off; latency is timed on a stratified sample because that is what a sample +is for. -Run: python -m bench.leg_check [--probes 300] +Run: python -m bench.leg_check [--probes 300] # --probes sizes the timing sample only Writes: bench/reports/LEGS.md, bench/reports/leg_check.json """ @@ -40,9 +43,6 @@ ENCODE_SAMPLE = 200 # The page the product asks for by default; measure() checks it against Settings, so "as shipped" cannot rot into a hypothesis. SHIPPED_TOP_K = 24 -# The modality whose questions carry a question type; measured whole, because 15 probes of 150 cannot separate six categories. -MEMORY_MODALITY = "conversation" - # (label, atoms asked, dense leg, picture leg); the picture leg answers only on a thin page, which is the product's rule. CONFIGS: list[dict[str, Any]] = [ {"label": "words only", "top_k": 10, "dense": False, "visual": False}, @@ -232,16 +232,18 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: f"SHIPPED_TOP_K={SHIPPED_TOP_K}; update the rows before trusting the 'as shipped' label" ) - chosen = scale_probes(load_queries(), probes) - answerable = [query for query in chosen if query.gold] + every = load_queries() + answerable = [query for query in every if query.gold] if not answerable: raise SystemExit("the frozen suite carries no answerable query; nothing to score") - print(f"{len(answerable)} answerable probes", flush=True) + timed = {query.query_id for query in scale_probes(every, probes) if query.gold} + if not timed: + raise SystemExit( + f"--probes {probes} picked no query to time: the sample strides the suite by modality, " + "so a small count can skip one and the latency row would be a single modality" + ) + print(f"{len(answerable)} answerable queries scored, {len(timed)} of them timed", flush=True) times = source_times() - memory = [query for query in load_queries() if query.modality == MEMORY_MODALITY and query.gold] - if not memory: - raise SystemExit(f"the frozen suite carries no answerable {MEMORY_MODALITY} query; the memory tables would be empty") - print(f"{len(memory)} answerable {MEMORY_MODALITY} queries, measured whole beside the probes", flush=True) vectors = _dense_vectors(home / "data" / "ely_eye.sqlite") source_schema = _schema(home) @@ -287,13 +289,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: for query, ranking in zip(answerable, rankings, strict=True) ] summary = aggregate(scores) - timing = latency(timings) - memory_rankings, _, _ = rank_all(service, ids, memory, int(config["top_k"])) - memory_scores = [ - score_query(query.query_id, ranking, set(query.gold), query.holder) - for query, ranking in zip(memory, memory_rankings, strict=True) - ] - memory_summary = aggregate(memory_scores) + timing = latency([t for query, t in zip(answerable, timings, strict=True) if query.query_id in timed]) arms.append( { "label": config["label"], @@ -308,14 +304,6 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: "p95_seconds": timing["p95"], "median_distinct_sources": statistics.median(distinct), **_groups(answerable, scores, rankings, times), - "memory": { - "queries": len(memory), - "recall_at_10": memory_summary["recall_at_10"], - "mrr": memory_summary["mrr"], - "hit_at_10": memory_summary["hit_at_10"], - "wilson_lower": memory_summary["hit_at_10_wilson_lower"], - **_groups(memory, memory_scores, memory_rankings, times), - }, } ) print( @@ -333,9 +321,8 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: "dense_vectors": vectors, "picture_index": picture, "encode_rate": rate, - "probes": len(answerable), - "memory_queries": len(memory), - "memory_modality": MEMORY_MODALITY, + "queries": len(answerable), + "timed_queries": len(timed), "arms": arms, "machine": machine_facts(), } diff --git a/bench/leg_report.py b/bench/leg_report.py index 40596d2..f33ae67 100644 --- a/bench/leg_report.py +++ b/bench/leg_report.py @@ -32,9 +32,9 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: ) counts = report["arms"][0]["modality_counts"] - def memory_table(title: str, field: str, sizes_field: str) -> list[str]: - """A table over the question types the memory set labels; unlabelled sets stay out of it.""" - rows = [arm["memory"] for arm in report["arms"]] + def category_table(title: str, field: str, sizes_field: str) -> list[str]: + """A table over the question types the set records; unlabelled sets stay out of it.""" + rows = report["arms"] names = sorted({name for row in rows for name in row[field] if name != "unlabelled"}) if not names: return [] @@ -79,11 +79,12 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: "sits outside the measured latency, and the frozen baseline itself holds no " "vectors, so a dense arm there would rank an empty index." ), - f"- {report['probes']} answerable queries, each asked as the holder the frozen set assigns it", ( - f"- memory subset: every answerable {report['memory_modality']} query " - f"({report['memory_queries']}) measured whole, because a 15-probe sample cannot " - "separate six question types; the two tables further down use that subset, not the sample" + f"- {report['queries']:,} answerable queries scored whole, each asked as the " + f"holder the frozen set assigns it; latency is timed on a stratified sample of " + f"{report['timed_queries']}, because a sample sized for timing is not sized for a " + "modality's recall — the 300-probe sample this harness used before read the " + "screenshot modality 0.818 where all 320 of its answerable queries read 0.659" ), *( [] @@ -121,9 +122,9 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: "", f"_Query counts per modality: {', '.join(f'{name} {count}' for name, count in counts.items())}._", "", - *memory_table("recall@10 by question type, every memory query measured", "by_category", "category_counts"), + *category_table("recall@10 by question type", "by_category", "category_counts"), "", - *memory_table( + *category_table( "the newest of the gold memories came back first", "newest_gold_first_by_category", "newest_gold_first_counts", diff --git a/bench/tests/test_leg_check.py b/bench/tests/test_leg_check.py index 2146582..1b3b94b 100644 --- a/bench/tests/test_leg_check.py +++ b/bench/tests/test_leg_check.py @@ -33,18 +33,10 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None: "by_modality": {"code": 0.953, "pdf": 0.849}, "modality_counts": {"code": 128, "pdf": 86}, "newest_gold_first": 0.44, - "memory": { - "queries": 150, - "recall_at_10": 0.965, - "mrr": 0.875, - "hit_at_10": 0.98, - "wilson_lower": 0.921, - "by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947}, - "category_counts": {"knowledge-update": 25, "temporal-reasoning": 25}, - "newest_gold_first": 0.4, - "newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526}, - "newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21}, - }, + "by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947}, + "category_counts": {"knowledge-update": 25, "temporal-reasoning": 25}, + "newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526}, + "newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21}, } report: dict[str, Any] = { "commit": "abcdef123456", @@ -55,9 +47,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None: "dense_vectors": 153857, "picture_index": {"path": "bench/runs/leg_check_index", "atoms": 53668, "dim": 512, "bytes": 109912192}, "encode_rate": {"pictures": 200, "seconds": 10.0, "per_second": 20.0}, - "probes": 300, - "memory_queries": 150, - "memory_modality": "conversation", + "queries": 2928, + "timed_queries": 300, "arms": [arm], "machine": { "system": "Linux", @@ -83,8 +74,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None: "knowledge-update" in text and "1.000" in text and "0.40 (19)" in text ), "the two question types the contract names need their own recall and their own n" assert ( - "every answerable conversation query (150) measured whole" in text - ), "a 15-probe sample cannot separate six question types, and the report must say which set the table used" + "2,928 answerable queries scored whole" in text and "stratified sample of 300" in text + ), "recall is scored over the whole suite and only latency is sampled, and the report must say so" assert "| 0.44 |" in text, "the probe table needs its own newest-first column" assert ( "arrives at v7 and the product migrates it when it opens, so every arm ran on v8" in text