Time every modality and ask which memory came back first

One stride over file order, truncated to the probe count, dropped the
tail of the suite: a 300-probe sample held 128 code, 86 pdf, 36
screenshot, 26 image and 24 video queries and no conversation query at
all, so every report built from it measured five of six modalities.
The budget is now split by suite weight with a floor of one probe per
modality, and a test pins that the frozen sample covers all six.

The set's question type rides along on each query, and a second metric
asks the question recall@10 cannot: when the answer lives in several
dated memories, did the newest come back first? A knowledge-update
question scores 1.000 on recall while the ranking still leads with the
value the speaker has since replaced, and that is the failure the next
task is meant to fix.

bench/tests/test_leg_check.py

@@ -32,6 +32,11 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
         "median_distinct_sources": 4,
         "by_modality": {"code": 0.953, "pdf": 0.849},
         "modality_counts": {"code": 128, "pdf": 86},
+        "by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947, "unlabelled": 0.84},
+        "category_counts": {"knowledge-update": 25, "temporal-reasoning": 19, "unlabelled": 256},
+        "newest_gold_first": 0.44,
+        "newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526},
+        "newest_gold_first_counts": {"knowledge-update": 25, "temporal-reasoning": 19},
     }
     report: dict[str, Any] = {
         "commit": "abcdef123456",
@@ -63,4 +68,8 @@ def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
     assert (
         "53,668 atoms at 512 dimensions" in text and "44.7" in text
     ), "the picture index needs its shape in the report, and its build cost derived from a measured rate"
+    assert (
+        "knowledge-update" in text and "1.000" in text and "0.40 (25)" in text
+    ), "the two question types the contract names need their own recall and their own n"
+    assert "unlabelled" not in text, "an unlabelled bucket would hide which set a number came from"
     assert "abcdef123456" in text and "load_average" not in text

bench/tests/test_metrics.py

@@ -9,6 +9,7 @@ from bench.metrics import (
     chance_recall_at_k,
     first_hit_rank,
     ndcg_at_10,
+    newest_gold_first,
     recall_at_k,
     reciprocal_rank,
     score_query,
@@ -101,3 +102,28 @@ def test_aggregate_separates_answerable_from_unanswerable() -> None:
 def test_aggregate_needs_answerable_queries() -> None:
     with pytest.raises(SystemExit):
         aggregate([score_query("none", [], set())])
+
+
+TIMES = {
+    "old": "2023-01-01T00:00:00+00:00",
+    "new": "2024-06-01T00:00:00+00:00",
+    "undated": "",
+}
+
+
+def test_a_recall_hit_on_an_outdated_memory_is_not_reported_as_the_answer() -> None:
+    """recall@10 scores both orders the same; a replaced value is not the answer."""
+    assert newest_gold_first(["old", "new"], {"old", "new"}, TIMES) is False
+    assert newest_gold_first(["new", "old"], {"old", "new"}, TIMES) is True
+
+
+def test_a_question_with_one_dated_memory_has_no_newest_answer() -> None:
+    assert newest_gold_first(["old"], {"old"}, TIMES) is None
+    assert newest_gold_first(["old", "undated"], {"old", "undated"}, {"old": TIMES["old"]}) is None
+
+
+def test_a_ranking_that_returned_none_of_the_dated_memories_is_not_counted() -> None:
+    assert newest_gold_first(["x", "y"], {"old", "new"}, TIMES) is None
+    assert (
+        newest_gold_first(["old", "x"], {"old", "new"}, TIMES) is False
+    ), "a newest memory that never came back did not come back first"

bench/tests/test_query_set.py

@@ -0,0 +1,83 @@
+"""Who gets timed: the probe sample must cover the suite it claims to measure."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+import pytest
+
+from bench.query_set import Query, load_queries, scale_probes
+
+
+def _suite(sizes: dict[str, int]) -> list[Query]:
+    queries: list[Query] = []
+    for modality, size in sizes.items():
+        queries.extend(
+            Query(
+                query_id=f"{modality}-{index}",
+                text="t",
+                modality=modality,
+                gold=frozenset({f"{modality}-s{index}"}),
+                holder="h",
+                answerable=True,
+                as_of=None,
+            )
+            for index in range(size)
+        )
+    return queries
+
+
+SIZES = {"code": 1237, "pdf": 766, "ui_screenshot": 320, "image": 240, "video": 215, "conversation": 150}
+
+
+def test_a_probe_sample_covers_every_modality_it_claims_to_measure() -> None:
+    probes = scale_probes(_suite(SIZES), 300)
+    mix = Counter(query.modality for query in probes)
+    assert len(probes) == 300
+    assert set(mix) == set(SIZES), f"a modality missing from the sample is unmeasured: {dict(mix)}"
+    assert mix["code"] > mix["conversation"], "the sample should still lean toward the big modalities"
+    assert probes == scale_probes(_suite(SIZES), 300), "the curve must be reproducible across runs"
+
+
+def test_the_frozen_sample_holds_a_conversation_query() -> None:
+    """The suite's smallest modality is the one a plain stride over file order used to drop."""
+    mix = Counter(query.modality for query in scale_probes(load_queries(), 300))
+    assert set(mix) == set(SIZES), f"the frozen sample is missing {sorted(set(SIZES) - set(mix))}: {dict(mix)}"
+
+
+def test_a_sample_smaller_than_the_modality_count_drops_the_smallest_pools() -> None:
+    mix = Counter(query.modality for query in scale_probes(_suite(SIZES), 2))
+    assert set(mix) == {"code", "pdf"}
+    assert sum(mix.values()) == 2
+
+
+def test_a_query_carries_the_category_its_set_recorded() -> None:
+    """LongMemEval labels each question's kind; the acceptance measurement groups by it."""
+    assert {query.category for query in load_queries(["conversation"])} == {
+        "knowledge-update",
+        "multi-session",
+        "single-session-assistant",
+        "single-session-preference",
+        "single-session-user",
+        "temporal-reasoning",
+    }
+    assert {query.category for query in load_queries(["pdf"])} == {None}, "a set without types stays ungrouped"
+
+
+def test_an_unanswerable_query_is_never_timed() -> None:
+    queries = [
+        Query(
+            query_id=f"q{index}",
+            text="t",
+            modality="pdf",
+            gold=frozenset() if index % 2 else frozenset({f"s{index}"}),
+            holder="h",
+            answerable=index % 2 == 0,
+            as_of=None,
+        )
+        for index in range(20)
+    ]
+    probes = scale_probes(queries, 4)
+    assert len(probes) == 4 and all(query.answerable for query in probes)
+    with pytest.raises(SystemExit, match="needs answerable"):
+        scale_probes([query for query in queries if not query.answerable], 4)