Time every modality and ask which memory came back first

One stride over file order, truncated to the probe count, dropped the
tail of the suite: a 300-probe sample held 128 code, 86 pdf, 36
screenshot, 26 image and 24 video queries and no conversation query at
all, so every report built from it measured five of six modalities.
The budget is now split by suite weight with a floor of one probe per
modality, and a test pins that the frozen sample covers all six.

The set's question type rides along on each query, and a second metric
asks the question recall@10 cannot: when the answer lives in several
dated memories, did the newest come back first? A knowledge-update
question scores 1.000 on recall while the ranking still leads with the
value the speaker has since replaced, and that is the failure the next
task is meant to fix.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-09 06:14:04 -04:00
1 parent 246dae949c
commit a6a1ef05f0
6 files changed
+265 -15

No files matched your search

+26
View File
@@ -9,6 +9,7 @@ from bench.metrics import (
chance_recall_at_k,
first_hit_rank,
ndcg_at_10,
newest_gold_first,
recall_at_k,
reciprocal_rank,
score_query,
@@ -101,3 +102,28 @@ def test_aggregate_separates_answerable_from_unanswerable() -> None:
def test_aggregate_needs_answerable_queries() -> None:
with pytest.raises(SystemExit):
aggregate([score_query("none", [], set())])
TIMES = {
"old": "2023-01-01T00:00:00+00:00",
"new": "2024-06-01T00:00:00+00:00",
"undated": "",
}
def test_a_recall_hit_on_an_outdated_memory_is_not_reported_as_the_answer() -> None:
"""recall@10 scores both orders the same; a replaced value is not the answer."""
assert newest_gold_first(["old", "new"], {"old", "new"}, TIMES) is False
assert newest_gold_first(["new", "old"], {"old", "new"}, TIMES) is True
def test_a_question_with_one_dated_memory_has_no_newest_answer() -> None:
assert newest_gold_first(["old"], {"old"}, TIMES) is None
assert newest_gold_first(["old", "undated"], {"old", "undated"}, {"old": TIMES["old"]}) is None
def test_a_ranking_that_returned_none_of_the_dated_memories_is_not_counted() -> None:
assert newest_gold_first(["x", "y"], {"old", "new"}, TIMES) is None
assert (
newest_gold_first(["old", "x"], {"old", "new"}, TIMES) is False
), "a newest memory that never came back did not come back first"