Score recall over the whole suite and time only a stratified sample

The 300-probe sample this harness scored by modality read the screenshot
modality 0.818 where all 320 of its answerable queries read 0.659, because
striding a file-ordered list and truncating at 300 reshuffles the modality
mix. Recall is now scored over every answerable query (2,928) and latency
over a stratified sample, which is what latency needs and what recall does
not. The separate conversation sub-pass is gone: conversation is inside the
suite, and by_category already carries its six LongMemEval types.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-10 01:59:25 -04:00
1 parent 958f635a66
commit e81923f62e
3 files changed
+37 -58

No files matched your search

+8 -17
View File
@@ -33,18 +33,10 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
"by_modality": {"code": 0.953, "pdf": 0.849},
"modality_counts": {"code": 128, "pdf": 86},
"newest_gold_first": 0.44,
"memory": {
"queries": 150,
"recall_at_10": 0.965,
"mrr": 0.875,
"hit_at_10": 0.98,
"wilson_lower": 0.921,
"by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947},
"category_counts": {"knowledge-update": 25, "temporal-reasoning": 25},
"newest_gold_first": 0.4,
"newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526},
"newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21},
},
"by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947},
"category_counts": {"knowledge-update": 25, "temporal-reasoning": 25},
"newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526},
"newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21},
}
report: dict[str, Any] = {
"commit": "abcdef123456",
@@ -55,9 +47,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
"dense_vectors": 153857,
"picture_index": {"path": "bench/runs/leg_check_index", "atoms": 53668, "dim": 512, "bytes": 109912192},
"encode_rate": {"pictures": 200, "seconds": 10.0, "per_second": 20.0},
"probes": 300,
"memory_queries": 150,
"memory_modality": "conversation",
"queries": 2928,
"timed_queries": 300,
"arms": [arm],
"machine": {
"system": "Linux",
@@ -83,8 +74,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
"knowledge-update" in text and "1.000" in text and "0.40 (19)" in text
), "the two question types the contract names need their own recall and their own n"
assert (
"every answerable conversation query (150) measured whole" in text
), "a 15-probe sample cannot separate six question types, and the report must say which set the table used"
"2,928 answerable queries scored whole" in text and "stratified sample of 300" in text
), "recall is scored over the whole suite and only latency is sampled, and the report must say so"
assert "| 0.44 |" in text, "the probe table needs its own newest-first column"
assert (
"arrives at v7 and the product migrates it when it opens, so every arm ran on v8" in text