diff --git a/bench/reports/BASELINE_EXTERNAL.md b/bench/reports/BASELINE_EXTERNAL.md index 61ca11c..de4afa8 100644 --- a/bench/reports/BASELINE_EXTERNAL.md +++ b/bench/reports/BASELINE_EXTERNAL.md @@ -1,16 +1,17 @@ # BASELINE_EXTERNAL: memory counterparts on the same corpus -- generated 2026-10-07T17:26:07+00:00 at commit c4e2982506fa +- generated 2026-10-07T19:16:08+00:00 at commit 3d792417cfd0 - 5308 queries (2928 answerable) against 50789 sources -- machine: 32 cores, load average 1.5 when the report was written +- machine: 32 cores, load average 2.9 when the report was written - extraction is lexical for both stores; Mem0's real add() and Zep's graph builder call a generator model, which this round does not have +- chance: random guessing scores recall@10 0.00020 over the whole corpus and 0.3686 inside one holder's memory (median 43 sources per holder); read each row against its own floor | store | items | closed | recall@10 | hit@10 | hit@10 Wilson 95% lower | MRR | abstain (answerable) | abstain (unanswerable) | p50 s | p95 s | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| mem0_flat [holder scope] | 403859 | 0 | 0.866 | 0.867 | 0.855 | 0.629 | 0.000 | 0.000 | 0.084 | 0.133 | -| mem0_flat [whole corpus] | 403859 | 0 | 0.412 | 0.416 | 0.398 | 0.283 | 0.000 | 0.000 | 0.090 | 0.174 | -| zep_temporal [holder scope] | 403859 | 27072 | 0.866 | 0.867 | 0.854 | 0.627 | 0.000 | 0.058 | 0.088 | 0.111 | -| zep_temporal [whole corpus] | 403859 | 27072 | 0.429 | 0.433 | 0.415 | 0.297 | 0.000 | 0.000 | 0.086 | 0.107 | +| mem0_flat [holder scope] | 403859 | 0 | 0.866 | 0.867 | 0.855 | 0.629 | 0.000 | 0.000 | 0.083 | 0.105 | +| mem0_flat [whole corpus] | 403859 | 0 | 0.412 | 0.416 | 0.398 | 0.283 | 0.000 | 0.000 | 0.086 | 0.112 | +| zep_temporal [holder scope] | 403859 | 27072 | 0.866 | 0.867 | 0.854 | 0.627 | 0.000 | 0.058 | 0.084 | 0.113 | +| zep_temporal [whole corpus] | 403859 | 27072 | 0.429 | 0.433 | 0.415 | 0.297 | 0.000 | 0.000 | 0.094 | 0.121 | n per modality: {"code": 2474, "conversation": 300, "image": 480, "pdf": 984, "ui_screenshot": 640, "video": 430} diff --git a/bench/reports/CURRENT.md b/bench/reports/CURRENT.md new file mode 100644 index 0000000..5307ffd --- /dev/null +++ b/bench/reports/CURRENT.md @@ -0,0 +1,62 @@ +# CURRENT: what the product retrieves at this build + +- generated 2026-10-07T21:24:15+00:00 at commit dfc6cd4c2889 +- measured store schema: v5 +- corpus 50278 sources, 5308 queries (2928 answerable) +- models: dense `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2`, rerank `BAAI/bge-reranker-base` +- machine: 32 cores, 4 threads per query, load average 1.2 when the report was written + +| arm | recall@1 | recall@5 | recall@10 | nDCG@10 | MRR | hit@10 (95% lower) | p50 s | p95 s | +|---|---|---|---|---|---|---|---|---| +| chance (random ranking) | 0.000 | 0.000 | 0.000 | - | - | - | - | - | + +_re-timed on a stride sample with warm models; recall comes from the full pass (n=600 queries)_ +| bm25_only | 0.313 | 0.465 | 0.519 | 0.415 | 0.383 | 0.524 (0.505) | 0.631 | 2.768 | +| dense_only | 0.227 | 0.363 | 0.424 | 0.321 | 0.290 | 0.429 (0.411) | 0.081 | 0.134 | +| graph_only | 0.000 | 0.001 | 0.001 | 0.000 | 0.000 | 0.001 (0.000) | 0.055 | 0.095 | +| no_rerank | 0.342 | 0.510 | 0.565 | 0.452 | 0.418 | 0.571 (0.553) | 0.643 | 2.786 | +| fusion_rerank | 0.236 | 0.391 | 0.453 | 0.341 | 0.309 | 0.459 (0.441) | 7.000 | 13.247 | + +## recall@10 by modality + +| arm | pdf | image | ui_screenshot | video | code | conversation | +|---|---|---|---|---|---|---| +| bm25_only | 0.704 | 0.637 | 0.322 | 0.126 | 0.518 | 0.368 | +| dense_only | 0.698 | 0.481 | 0.075 | 0.228 | 0.382 | 0.304 | +| graph_only | 0.000 | 0.000 | 0.000 | 0.000 | 0.002 | 0.000 | +| no_rerank | 0.742 | 0.669 | 0.297 | 0.153 | 0.592 | 0.447 | +| fusion_rerank | 0.253 | 0.692 | 0.300 | 0.088 | 0.629 | 0.486 | + +## unanswerable queries (44.8% of the suite) + +- bm25_only: abstains on 0% of unanswerable queries, and its top hit comes from the query's own holder for 38% of them; the rest are confident answers pulled out of somebody else's memory +- dense_only: abstains on 0% of unanswerable queries, and its top hit comes from the query's own holder for 25% of them; the rest are confident answers pulled out of somebody else's memory +- graph_only: abstains on 0% of unanswerable queries, and its top hit comes from the query's own holder for 0% of them; the rest are confident answers pulled out of somebody else's memory +- no_rerank: abstains on 0% of unanswerable queries, and its top hit comes from the query's own holder for 40% of them; the rest are confident answers pulled out of somebody else's memory +- fusion_rerank: abstains on 0% of unanswerable queries, and its top hit comes from the query's own holder for 34% of them; the rest are confident answers pulled out of somebody else's memory + +## the unmodified product path + +- RetrievalService.search: p50 0.68s, mean 1.08s, max 9.36s over 40 queries +- top-10 overlap with the fast BM25 path: 0.0%, identical order 0.0% +- its top-10 atoms cover only 7.7 distinct sources (min 1): the product ranks atoms and never diversifies, so ten slots fill with fragments of one document and the overlap with a source-level ranking is near zero + +## latency vs corpus size + +| sources | atoms | bm25_only p50/p95 s | dense_only p50/p95 s | +|---|---|---|---| +| 500 | 500 | 0.001/0.003 | 0.018/0.031 | +| 2000 | 2000 | 0.005/0.010 | 0.045/0.078 | +| 10000 | 10000 | 0.035/0.110 | 0.079/0.100 | +| 50000 | 165695 | 0.944/2.376 | 0.090/0.108 | + +_latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless_ + +_latency was measured at load average 1.2 on 32 cores; a busy machine inflates these figures, so read them as a ceiling_ + +## reading + +- recall counts a source as found only when it is the source a third party marked correct. +- abstention is the only honest number for unanswerable queries: a retriever that always + returns ten results never says it does not know. +- chance is the floor any of these arms must beat; a number near chance is not a capability. diff --git a/bench/reports/baseline_external.json b/bench/reports/baseline_external.json index c8b02b8..c132249 100644 --- a/bench/reports/baseline_external.json +++ b/bench/reports/baseline_external.json @@ -1,19 +1,26 @@ { + "chance": { + "median_holder_sources": 43.0, + "note": "expected recall@10 of a random ranking inside the candidate set each row is allowed to see", + "recall_at_10": 0.00019689302801787788, + "recall_at_10_holder_scope": 0.3685528769188164 + }, "meta": { "answerable": 2928, "extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator", - "generated_at": "2026-10-07T17:26:07+00:00", - "git_commit": "c4e2982506faaf350a4b06738923899fc44bdedd", + "generated_at": "2026-10-07T19:16:08+00:00", + "git_commit": "3d792417cfd0554cbe2bae0cfc098bb5bb07e28c", "limit_per_modality": null, "machine": { "cpus": 32, "load_average_1_5_15": [ - 1.47, - 2.81, - 10.33 + 2.85, + 10.0, + 14.84 ], "release": "6.18.40.1-microsoft-standard-WSL2", - "system": "Linux" + "system": "Linux", + "threads_per_query": 4 }, "queries": 5308, "queries_by_modality": { @@ -118,10 +125,10 @@ "closed_items": 0, "items": 403859, "latency_seconds": { - "max": 7.547149860001809, - "mean": 0.09187759584445908, - "p50": 0.08432659949903609, - "p95": 0.13331470334669573, + "max": 5.168096734996652, + "mean": 0.08254052409894046, + "p50": 0.08343072250136174, + "p95": 0.10547906984866127, "queries": 5308 }, "overall": { @@ -230,10 +237,10 @@ "closed_items": 0, "items": 403859, "latency_seconds": { - "max": 2.318251260003308, - "mean": 0.10140667538014644, - "p50": 0.09026374550012406, - "p95": 0.17443375029943126, + "max": 0.1391175699973246, + "mean": 0.08744079960348271, + "p50": 0.0859174314973643, + "p95": 0.11227948825326166, "queries": 5308 }, "overall": { @@ -342,10 +349,10 @@ "closed_items": 27072, "items": 403859, "latency_seconds": { - "max": 18.72793256899604, - "mean": 0.09218696485981677, - "p50": 0.08754261549984221, - "p95": 0.11125843500121846, + "max": 1.9491578790039057, + "mean": 0.08603176019042821, + "p50": 0.0841971809968527, + "p95": 0.11250105440230979, "queries": 5308 }, "overall": { @@ -454,10 +461,10 @@ "closed_items": 27072, "items": 403859, "latency_seconds": { - "max": 0.2064202700057649, - "mean": 0.08776056289499023, - "p50": 0.08638653549860464, - "p95": 0.10662773229960291, + "max": 0.17060414699517423, + "mean": 0.0951456186994907, + "p50": 0.09420693750143982, + "p95": 0.12111507409863406, "queries": 5308 }, "overall": { diff --git a/bench/reports/current.json b/bench/reports/current.json new file mode 100644 index 0000000..d6f5f6d --- /dev/null +++ b/bench/reports/current.json @@ -0,0 +1,671 @@ +{ + "arms": [ + { + "arm": "bm25_only", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5181891673403395, + "hit_at_10_wilson_lower": 0.4903310330926623, + "mrr": 0.36655150068650477, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.4030722360859963, + "recall_at_1": 0.29345189975747776, + "recall_at_10": 0.5181891673403395, + "recall_at_5": 0.46160064672594986, + "top_hit_from_holder_unanswerable": 0.629749393694422 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.4533333333333333, + "hit_at_10_wilson_lower": 0.37582498607927695, + "mrr": 0.2632063492063492, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.2739998550294506, + "recall_at_1": 0.16333333333333333, + "recall_at_10": 0.36822222222222223, + "recall_at_5": 0.31222222222222223, + "top_hit_from_holder_unanswerable": 0.006666666666666667 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6458333333333334, + "hit_at_10_wilson_lower": 0.5834633143794732, + "mrr": 0.5017857142857143, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.5293922693217483, + "recall_at_1": 0.42083333333333334, + "recall_at_10": 0.6375, + "recall_at_5": 0.575, + "top_hit_from_holder_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.7036553524804178, + "hit_at_10_wilson_lower": 0.6703659725843993, + "mrr": 0.564516246011024, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.5982115197161956, + "recall_at_1": 0.4934725848563969, + "recall_at_10": 0.7036553524804178, + "recall_at_5": 0.6540469973890339, + "top_hit_from_holder_unanswerable": 0.5688073394495413 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.321875, + "hit_at_10_wilson_lower": 0.27306019860147723, + "mrr": 0.1832576884920635, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.21641192936033926, + "recall_at_1": 0.11875, + "recall_at_10": 0.321875, + "recall_at_5": 0.2625, + "top_hit_from_holder_unanswerable": 0.015625 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.12558139534883722, + "hit_at_10_wilson_lower": 0.08776042446405018, + "mrr": 0.08080103359173127, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.09134653357120204, + "recall_at_1": 0.06046511627906977, + "recall_at_10": 0.12558139534883722, + "recall_at_5": 0.10232558139534884, + "top_hit_from_holder_unanswerable": 0.004651162790697674 + } + }, + "latency_seconds": { + "max": 10.484337914007483, + "mean": 0.9426860555750318, + "note": "re-timed on a stride sample with warm models; recall comes from the full pass", + "p50": 0.63090720949549, + "p95": 2.7677196743439674, + "queries": 600 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5235655737704918, + "hit_at_10_wilson_lower": 0.5054560637114823, + "mrr": 0.38311743754879, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.41457507689403594, + "recall_at_1": 0.31335382513661203, + "recall_at_10": 0.5185223132969035, + "recall_at_5": 0.4654485428051002, + "top_hit_from_holder_unanswerable": 0.38235294117647056 + } + }, + { + "arm": "dense_only", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.38156831042845596, + "hit_at_10_wilson_lower": 0.3549039694490394, + "mrr": 0.23736478423220542, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.2717431059401204, + "recall_at_1": 0.17623282134195634, + "recall_at_10": 0.38156831042845596, + "recall_at_5": 0.3193209377526273, + "top_hit_from_holder_unanswerable": 0.37510105092966856 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.38666666666666666, + "hit_at_10_wilson_lower": 0.31249116779192654, + "mrr": 0.2273148148148148, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.2312801291223675, + "recall_at_1": 0.12666666666666668, + "recall_at_10": 0.3042222222222222, + "recall_at_5": 0.26466666666666666, + "top_hit_from_holder_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.4875, + "hit_at_10_wilson_lower": 0.4249589194528996, + "mrr": 0.3143535052910053, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.35184243019735995, + "recall_at_1": 0.21875, + "recall_at_10": 0.48125, + "recall_at_5": 0.42291666666666666, + "top_hit_from_holder_unanswerable": 0.0125 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6984334203655352, + "hit_at_10_wilson_lower": 0.6650089879505938, + "mrr": 0.5372964068133781, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.5759767947831819, + "recall_at_1": 0.4608355091383812, + "recall_at_10": 0.6984334203655352, + "recall_at_5": 0.6292428198433421, + "top_hit_from_holder_unanswerable": 0.5779816513761468 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.075, + "hit_at_10_wilson_lower": 0.05091491480619657, + "mrr": 0.030979662698412697, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.04122199724982832, + "recall_at_1": 0.015625, + "recall_at_10": 0.075, + "recall_at_5": 0.046875, + "top_hit_from_holder_unanswerable": 0.0 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.22790697674418606, + "hit_at_10_wilson_lower": 0.1769011082994195, + "mrr": 0.1167921742340347, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.14219297062759328, + "recall_at_1": 0.08372093023255814, + "recall_at_10": 0.22790697674418606, + "recall_at_5": 0.14418604651162792, + "top_hit_from_holder_unanswerable": 0.013953488372093023 + } + }, + "latency_seconds": { + "max": 1.5199895569967339, + "mean": 0.09428343026663545, + "note": "re-timed on a stride sample with warm models; recall comes from the full pass", + "p50": 0.08063396950092283, + "p95": 0.13402395970297218, + "queries": 600 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.428620218579235, + "hit_at_10_wilson_lower": 0.41080015838413203, + "mrr": 0.29021692579581926, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.32112062065315755, + "recall_at_1": 0.22728825136612021, + "recall_at_10": 0.423884335154827, + "recall_at_5": 0.36345628415300546, + "top_hit_from_holder_unanswerable": 0.2504201680672269 + } + }, + { + "arm": "graph_only", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0016168148746968471, + "hit_at_10_wilson_lower": 0.0004435009881340089, + "mrr": 0.0009700889248181083, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.001121142123875943, + "recall_at_1": 0.0008084074373484236, + "recall_at_10": 0.0016168148746968471, + "recall_at_5": 0.0016168148746968471, + "top_hit_from_holder_unanswerable": 0.0 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "top_hit_from_holder_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "top_hit_from_holder_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "top_hit_from_holder_unanswerable": 0.0 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "top_hit_from_holder_unanswerable": 0.0 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "top_hit_from_holder_unanswerable": 0.0 + } + }, + "latency_seconds": { + "max": 0.11273866199917393, + "mean": 0.06529701035842057, + "note": "re-timed on a stride sample with warm models; recall comes from the full pass", + "p50": 0.05544837899287813, + "p95": 0.09468995315801294, + "queries": 600 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0006830601092896175, + "hit_at_10_wilson_lower": 0.00018733970917281023, + "mrr": 0.0004098360655737705, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.0004736519150391194, + "recall_at_1": 0.00034153005464480874, + "recall_at_10": 0.0006830601092896175, + "recall_at_5": 0.0006830601092896175, + "top_hit_from_holder_unanswerable": 0.0 + } + }, + { + "arm": "no_rerank", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5917542441390461, + "hit_at_10_wilson_lower": 0.5641209696430517, + "mrr": 0.4132014217705406, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.4559664301805668, + "recall_at_1": 0.3314470493128537, + "recall_at_10": 0.5917542441390461, + "recall_at_5": 0.52303961196443, + "top_hit_from_holder_unanswerable": 0.666127728375101 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.54, + "hit_at_10_wilson_lower": 0.46023821464666953, + "mrr": 0.3229312169312169, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.3349165804130411, + "recall_at_1": 0.19, + "recall_at_10": 0.44711111111111107, + "recall_at_5": 0.38155555555555554, + "top_hit_from_holder_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.675, + "hit_at_10_wilson_lower": 0.6133905629314187, + "mrr": 0.5472437169312169, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.5716072495855514, + "recall_at_1": 0.4791666666666667, + "recall_at_10": 0.66875, + "recall_at_5": 0.6, + "top_hit_from_holder_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.741514360313316, + "hit_at_10_wilson_lower": 0.7093596270841578, + "mrr": 0.5952085664553027, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.6306490300771556, + "recall_at_1": 0.5195822454308094, + "recall_at_10": 0.741514360313316, + "recall_at_5": 0.695822454308094, + "top_hit_from_holder_unanswerable": 0.6146788990825688 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.296875, + "hit_at_10_wilson_lower": 0.24946567415196566, + "mrr": 0.1761359126984127, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.20522256852542595, + "recall_at_1": 0.11875, + "recall_at_10": 0.296875, + "recall_at_5": 0.259375, + "top_hit_from_holder_unanswerable": 0.009375 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.15348837209302327, + "hit_at_10_wilson_lower": 0.11142806397476775, + "mrr": 0.0915891472868217, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.10659041518078256, + "recall_at_1": 0.06046511627906977, + "recall_at_10": 0.15348837209302327, + "recall_at_5": 0.13953488372093023, + "top_hit_from_holder_unanswerable": 0.004651162790697674 + } + }, + "latency_seconds": { + "max": 10.533955075996346, + "mean": 0.9619427272766673, + "note": "re-timed on a stride sample with warm models; recall comes from the full pass", + "p50": 0.6425201155034301, + "p95": 2.786401089788338, + "queries": 600 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5706967213114754, + "hit_at_10_wilson_lower": 0.552686960572952, + "mrr": 0.4176547998525457, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.4518849109723208, + "recall_at_1": 0.34238387978142076, + "recall_at_10": 0.5654257741347906, + "recall_at_5": 0.510325591985428, + "top_hit_from_holder_unanswerable": 0.4042016806722689 + } + }, + { + "arm": "fusion_rerank", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6289409862570736, + "hit_at_10_wilson_lower": 0.6016596375208901, + "mrr": 0.4450988695127741, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.48964513720914205, + "recall_at_1": 0.35084882780921584, + "recall_at_10": 0.6289409862570736, + "recall_at_5": 0.5675020210185934, + "top_hit_from_holder_unanswerable": 0.6313662085691188 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5866666666666667, + "hit_at_10_wilson_lower": 0.5066585675662262, + "mrr": 0.44420105820105815, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.41993124265105486, + "recall_at_1": 0.28, + "recall_at_10": 0.4862222222222222, + "recall_at_5": 0.42911111111111105, + "top_hit_from_holder_unanswerable": 0.006666666666666667 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.7, + "hit_at_10_wilson_lower": 0.6392449572501318, + "mrr": 0.5878356481481481, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.6098813565843775, + "recall_at_1": 0.5208333333333334, + "recall_at_10": 0.6916666666666667, + "recall_at_5": 0.6520833333333333, + "top_hit_from_holder_unanswerable": 0.016666666666666666 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.25326370757180156, + "hit_at_10_wilson_lower": 0.223750502086736, + "mrr": 0.1029870695014298, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.13796388574710278, + "recall_at_1": 0.05221932114882506, + "recall_at_10": 0.25326370757180156, + "recall_at_5": 0.16579634464751958, + "top_hit_from_holder_unanswerable": 0.0871559633027523 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.3, + "hit_at_10_wilson_lower": 0.25240562491350604, + "mrr": 0.18429935515873014, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.21181060150559686, + "recall_at_1": 0.1375, + "recall_at_10": 0.3, + "recall_at_5": 0.25625, + "top_hit_from_holder_unanswerable": 0.03125 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.08837209302325581, + "hit_at_10_wilson_lower": 0.05730442391262212, + "mrr": 0.04147840531561462, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.0524917978865275, + "recall_at_1": 0.023255813953488372, + "recall_at_10": 0.08837209302325581, + "recall_at_5": 0.06511627906976744, + "top_hit_from_holder_unanswerable": 0.0 + } + }, + "latency_seconds": { + "max": 28.03755501300475, + "mean": 7.456139394386567, + "note": "re-timed on a stride sample with warm models; recall comes from the full pass", + "p50": 7.000474203505291, + "p95": 13.246865720593268, + "queries": 600 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.45867486338797814, + "hit_at_10_wilson_lower": 0.4406921315699621, + "mrr": 0.3091119405412438, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.34146096728664366, + "recall_at_1": 0.23565573770491804, + "recall_at_10": 0.4528460837887068, + "recall_at_5": 0.3913479052823315, + "top_hit_from_holder_unanswerable": 0.34243697478991597 + } + } + ], + "chance": { + "corpus_sources": 50278, + "recall_at_1": 1.9889414853415013e-05, + "recall_at_10": 0.00019889414853415012, + "recall_at_5": 9.944707426707506e-05 + }, + "live_product": { + "distinct_sources_in_top10": { + "max": 10, + "mean": 7.675, + "min": 1 + }, + "identical_top10": 0.0, + "max": 9.363575160998153, + "mean": 1.0766488450000906, + "p50": 0.6765933140050038, + "queries": 40, + "top10_overlap_with_bm25_only": 0.0, + "top_k": 10 + }, + "meta": { + "answerable": 2928, + "corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347", + "corpus_sources": 50278, + "generated_at": "2026-10-07T21:24:15+00:00", + "git_commit": "dfc6cd4c2889e872d3f7d983b18406e6ff7a622d", + "limit_per_modality": null, + "machine": { + "cpus": 32, + "load_average_1_5_15": [ + 1.21, + 1.55, + 2.28 + ], + "release": "6.18.40.1-microsoft-standard-WSL2", + "system": "Linux", + "threads_per_query": 4 + }, + "models": { + "dense": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "rerank": "BAAI/bge-reranker-base" + }, + "python": "3.14.4", + "queries": 5308, + "sets": { + "code": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e", + "conversation": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11", + "image": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128", + "pdf": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536", + "ui_screenshot": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5", + "video": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c" + }, + "store_schema_version_at_start": 5 + }, + "scale": [ + { + "atoms": 500, + "bm25_only": { + "p50": 0.001200532500661211, + "p95": 0.0033308441004919584 + }, + "dense_only": { + "p50": 0.018240716999571305, + "p95": 0.03141984814501483 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 500 + }, + { + "atoms": 2000, + "bm25_only": { + "p50": 0.0047556615027133375, + "p95": 0.009782735643602785 + }, + "dense_only": { + "p50": 0.044618372499826364, + "p95": 0.07758056885431869 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 2000 + }, + { + "atoms": 10000, + "bm25_only": { + "p50": 0.03480406300150207, + "p95": 0.11021578034851681 + }, + "dense_only": { + "p50": 0.07940824350225739, + "p95": 0.09990645345169469 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 10000 + }, + { + "atoms": 165695, + "bm25_only": { + "p50": 0.9441751839985955, + "p95": 2.3757928041479315 + }, + "dense_only": { + "p50": 0.08988298100302927, + "p95": 0.10847034014877863 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 50000 + } + ] +} \ No newline at end of file diff --git a/bench/run.py b/bench/run.py index 1886520..dab452f 100644 --- a/bench/run.py +++ b/bench/run.py @@ -41,7 +41,7 @@ from bench.systems import ( ROOT = SETS.parent REPORTS = ROOT / "reports" DB_HOME = ROOT / "runs" / "baseline" -LIVE_QUERIES = 40 # the product rebuilds BM25 per query, so a live pass costs minutes per query +LIVE_QUERIES = 300 # the acceptance bar names n>=300; the SQL path answers in well under a second PROGRESS_EVERY = 250 # an arm takes tens of minutes over 5,308 queries; silence looks like a hang SCALE_SIZES = (500, 2_000, 10_000, 50_000) SCALE_PROBES = 40 @@ -163,13 +163,19 @@ def live_check( top_k: int = 10, live_queries: int = LIVE_QUERIES, ) -> dict[str, Any]: - """Compare the unmodified RetrievalService with the fast BM25 path.""" + """Score the unmodified RetrievalService on its own, as its holder would ask it. + + The arms measure retrieval configurations; the acceptance bar is about the product. A query is + asked as the holder the frozen set assigns it, because a transactive memory is used that way. + """ probe = [query for query in queries if query.answerable] stride = max(1, len(probe) // live_queries) chosen = probe[::stride][:live_queries] if not chosen: raise SystemExit("live check needs answerable queries") - rankings, distinct, timings = live_product_rankings(DB_HOME, [query.text for query in chosen], top_k) + rankings, distinct, timings = live_product_rankings( + DB_HOME, [query.text for query in chosen], top_k, as_holders=[query.holder for query in chosen] + ) overlaps = [ len(set(fast_rankings[query.query_id]) & set(ranking)) / top_k for query, ranking in zip(chosen, rankings, strict=True) @@ -177,20 +183,30 @@ def live_check( ] if not overlaps: raise SystemExit("live check found no fast rankings to compare against") - return { - "queries": len(chosen), - "top_k": top_k, - "p50": sorted(timings)[len(timings) // 2], - "mean": sum(timings) / len(timings), - "max": max(timings), - "top10_overlap_with_bm25_only": sum(overlaps) / len(overlaps), - "identical_top10": sum(1 for overlap in overlaps if overlap == 1.0) / len(overlaps), - "distinct_sources_in_top10": { - "mean": sum(distinct) / len(distinct), - "min": min(distinct), - "max": max(distinct), - }, - } + scores = [ + score_query(query.query_id, ranking, set(query.gold), query.holder) + for query, ranking in zip(chosen, rankings, strict=True) + ] + metrics: dict[str, Any] = dict(aggregate(scores)) + ordered = sorted(timings) + metrics.update( + { + "queries": len(chosen), + "top_k": top_k, + "p50": ordered[len(ordered) // 2], + "p95": ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))], + "mean": sum(ordered) / len(ordered), + "max": max(ordered), + "top10_overlap_with_bm25_only": sum(overlaps) / len(overlaps), + "identical_top10": sum(1 for overlap in overlaps if overlap == 1.0) / len(overlaps), + "distinct_sources_in_top10": { + "mean": sum(distinct) / len(distinct), + "min": min(distinct), + "max": max(distinct), + }, + } + ) + return metrics def git_commit() -> str: @@ -203,7 +219,23 @@ def git_commit() -> str: return commit -def meta(corpus_sources: int, queries: Sequence[Query], limit: int | None = None) -> dict[str, Any]: +def store_schema_version(store: Path) -> int: + """The schema the measured store actually carries, read from the file.""" + import sqlite3 + + if not store.exists(): + raise SystemExit(f"{store}: the measured store is missing") + connection = sqlite3.connect(f"file:{store}?mode=ro", uri=True) + try: + row = connection.execute("PRAGMA user_version").fetchone() + finally: + connection.close() + return int(row[0]) + + +def meta( + corpus_sources: int, queries: Sequence[Query], limit: int | None = None, store: Path | None = None +) -> dict[str, Any]: manifest_path = SETS / "manifest.json" try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) @@ -220,6 +252,7 @@ def meta(corpus_sources: int, queries: Sequence[Query], limit: int | None = None "answerable": sum(1 for query in queries if query.answerable), "limit_per_modality": limit, "corpus_sources": corpus_sources, + "store_schema_version_at_start": None if store is None else store_schema_version(store), "models": {"dense": DENSE_MODEL, "rerank": RERANK_MODEL}, } @@ -240,12 +273,14 @@ def scale_arm_names(rows: Sequence[dict[str, Any]]) -> list[str]: return names -def write_markdown(report: dict[str, Any], path: Path) -> None: +def write_markdown(report: dict[str, Any], path: Path, name: str = "BASELINE") -> None: """Human-readable twin of the JSON report.""" + schema = report["meta"].get("store_schema_version_at_start") lines = [ - "# BASELINE: what the product retrieves today", + f"# {name}: what the product retrieves at this build", "", f"- generated {report['meta']['generated_at']} at commit {report['meta']['git_commit'][:12]}", + ("" if schema is None else f"- measured store schema: v{schema}"), ( f"- corpus {report['meta']['corpus_sources']} sources, {report['meta']['queries']} queries " f"({report['meta']['answerable']} answerable)" @@ -302,21 +337,26 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: live = report["live_product"] lines += [ "", - "## the unmodified product path", + "## the product path, asked as its holder", "", ( - f"- RetrievalService.search: p50 {live['p50']:.2f}s, mean {live['mean']:.2f}s, max {live['max']:.2f}s " - f"over {live['queries']} queries" + f"- RetrievalService.search as the query's own holder: recall@10 {live['recall_at_10']:.3f}, " + f"MRR {live['mrr']:.3f}, hit@10 {live['hit_at_10']:.3f} (95% lower {live['hit_at_10_wilson_lower']:.3f}) " + f"over n={live['queries']} answerable queries" + ), + ( + f"- latency p50 {live['p50']:.3f}s, p95 {live['p95']:.3f}s, mean {live['mean']:.3f}s, " + f"max {live['max']:.3f}s" ), ( f"- top-10 overlap with the fast BM25 path: {live['top10_overlap_with_bm25_only']:.1%}, " f"identical order {live['identical_top10']:.1%}" ), ( - f"- its top-{live['top_k']} atoms cover only " - f"{live['distinct_sources_in_top10']['mean']:.1f} distinct sources " + f"- its top-{live['top_k']} atoms cover " + f"{live['distinct_sources_in_top10']['mean']:.1f} distinct sources on average " f"(min {live['distinct_sources_in_top10']['min']}): the product ranks atoms and never diversifies, " - "so ten slots fill with fragments of one document and the overlap with a source-level ranking is near zero" + "so ten slots can fill with fragments of one document" ), ] if report.get("scale"): @@ -463,6 +503,11 @@ def main() -> int: help="re-time this many sampled queries with warm models instead of trusting cached per-query seconds", ) parser.add_argument("--out", type=Path, default=REPORTS) + parser.add_argument( + "--name", + default="BASELINE", + help="report name: writes NAME.md and name.json (BASELINE stays the frozen pre-P0 snapshot)", + ) parser.add_argument("--config", type=Path, help="JSON defaults for these flags (bench/configs/ablation.json)") args = parser.parse_args() if args.config: @@ -485,7 +530,7 @@ def main() -> int: print(f"{len(queries)} queries, {len(atoms)} atoms, {corpus_sources} sources", flush=True) report: dict[str, Any] = { - "meta": meta(corpus_sources, queries, args.limit), + "meta": meta(corpus_sources, queries, args.limit, store=DB_HOME / "data" / "ely_eye.sqlite"), "chance": chance_row(queries, corpus_sources), } # One pass over the queries: each signal is computed once and every arm is derived @@ -553,9 +598,10 @@ def main() -> int: report["scale"] = scale_curve(queries, atoms, [arm for arm in arms_requested if arm in ("bm25_only", "dense_only")]) args.out.mkdir(parents=True, exist_ok=True) - (args.out / "baseline.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8") - write_markdown(report, args.out / "BASELINE.md") - print(f"== wrote {args.out / 'baseline.json'} and {args.out / 'BASELINE.md'}", flush=True) + stem = args.name.lower() + (args.out / f"{stem}.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8") + write_markdown(report, args.out / f"{args.name}.md", name=args.name) + print(f"== wrote {args.out / (stem + '.json')} and {args.out / (args.name + '.md')}", flush=True) return 0 diff --git a/bench/stamp_holders.py b/bench/stamp_holders.py new file mode 100644 index 0000000..3cf0fcd --- /dev/null +++ b/bench/stamp_holders.py @@ -0,0 +1,78 @@ +"""Give every ingested atom the holder, scenario and event time the corpus index already states. + +The pre-P0 store was built before the memory model had those columns, so it holds 165,973 memories +nobody owns. The frozen corpus index names the owner, the setting and the moment for each source — +the same facts the counterpart and the gold scopes are written from — so the store can be brought +in line without re-running ingestion. + +Run: python -m bench.stamp_holders [--store bench/runs/baseline/data/ely_eye.sqlite] +""" + +from __future__ import annotations + +import argparse +import sqlite3 +from pathlib import Path + +from bench.sets_common import MODALITIES, SETS, read_jsonl, record_path + +REPO = Path(__file__).resolve().parent.parent +DEFAULT_STORE = REPO / "bench" / "runs" / "baseline" / "data" / "ely_eye.sqlite" + + +def corpus_facts() -> dict[str, tuple[str, str, str]]: + """Absolute path -> (holder, scenario, observed_at), as the frozen corpus index states them.""" + facts: dict[str, tuple[str, str, str]] = {} + for modality in MODALITIES: + for record in read_jsonl(SETS / "corpus" / f"{modality}.jsonl"): + for key in ("holder", "scenario", "observed_at"): + if not record.get(key): + raise SystemExit(f"{modality}: corpus record {record['source_id']} has no {key}") + facts[str(record_path(record).resolve())] = ( + str(record["holder"]), + str(record["scenario"]), + str(record["observed_at"]), + ) + if not facts: + raise SystemExit("no corpus index found, run python -m bench.build_sets first") + return facts + + +def stamp(store: Path) -> dict[str, int]: + """Write the corpus facts onto every atom; the store's own source ids stay untouched.""" + if not store.is_file(): + raise SystemExit(f"{store}: no baseline database, run python -m bench.ingest_baseline first") + facts = corpus_facts() + with sqlite3.connect(store) as connection: + rows = connection.execute("SELECT source_id, path FROM sources").fetchall() + if not rows: + raise SystemExit(f"{store}: the sources table is empty") + unknown = sorted(str(path) for _, path in rows if str(path) not in facts) + if unknown: + raise SystemExit(f"{store}: {len(unknown)} source paths are not corpus files, first {unknown[:3]}") + stamped_atoms = 0 + for source_id, path in rows: + holder, scenario, observed_at = facts[str(path)] + cursor = connection.execute( + "UPDATE atoms SET holder_id = ?, scenario = ?, event_time = ?, event_time_precision = 'instant' " + "WHERE source_id = ?", + (holder, scenario, observed_at, str(source_id)), + ) + stamped_atoms += int(cursor.rowcount) + left = int(connection.execute("SELECT count(*) FROM atoms WHERE holder_id = ''").fetchone()[0]) + return {"sources": len(rows), "atoms": stamped_atoms, "atoms_without_holder": left} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--store", type=Path, default=DEFAULT_STORE) + args = parser.parse_args() + counts = stamp(args.store) + print( + f"stamped {counts['atoms']} atoms across {counts['sources']} sources; " + f"{counts['atoms_without_holder']} atoms still have no holder" + ) + + +if __name__ == "__main__": + main() diff --git a/bench/systems.py b/bench/systems.py index 3b7928e..7f0aa09 100644 --- a/bench/systems.py +++ b/bench/systems.py @@ -18,11 +18,14 @@ import time from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np from bench.ingest_baseline import source_map + +if TYPE_CHECKING: + from ely_eye.db import Database from bench.sets_common import ROOT RUNS = ROOT / "runs" / "baseline" @@ -423,34 +426,67 @@ def _best_atom_text(atoms: Sequence[Atom], source_ids: Sequence[str]) -> list[tu return [(source_id, longest.get(source_id, "")) for source_id in source_ids] +def _corpus_ids(database: Database) -> dict[str, str]: + """Join the store's content-hash source ids to the ids the frozen sets call a source by. + + Without this join the product's hits and the gold answers share no name, and a correct answer + scores zero — which is what an earlier report recorded as "0% overlap with the fast path". + """ + mapping = source_map() + ids: dict[str, str] = {} + unknown: list[str] = [] + for source in database.list_sources(): + corpus_id = mapping.get(str(Path(source.path).resolve())) + if corpus_id is None: + unknown.append(source.source_id) + else: + ids[source.source_id] = corpus_id + if unknown: + raise SystemExit(f"{database.path}: {len(unknown)} store sources are not corpus files, first {unknown[:3]}") + return ids + + def live_product_rankings( - db_home: Path, queries: Sequence[str], top_k: int = 10 + db_home: Path, + queries: Sequence[str], + top_k: int = 10, + as_holders: Sequence[str | None] | None = None, ) -> tuple[list[list[str]], list[int], list[float]]: """Rankings from the unmodified RetrievalService, with per-query wall time. - The second list counts how many distinct sources the product's top-k atoms came - from: the product ranks atoms and never diversifies, so ten slots can be filled by - fragments of one document, and that is invisible once the hits are collapsed. + Rankings carry the benchmark's source ids, so they are comparable with the arms and with gold. + `as_holders` asks each query as its own holder, which is how a transactive memory is used: the + asker does not know where the answer is, only who might. + + The second list counts how many distinct sources the product's top-k atoms came from: the + product ranks atoms and never diversifies, so ten slots can be filled by fragments of one + document, and that is invisible once the hits are collapsed. """ from ely_eye.config import Settings from ely_eye.db import Database from ely_eye.retrieval import RetrievalService + from ely_eye.schemas import SearchFilters + if as_holders is not None and len(as_holders) != len(queries): + raise SystemExit(f"{len(as_holders)} holders for {len(queries)} queries") settings = Settings(home=db_home) - service = RetrievalService(settings=settings, database=Database(settings)) + database = Database(settings) + service = RetrievalService(settings=settings, database=database) + corpus_ids = _corpus_ids(database) rankings: list[list[str]] = [] distinct: list[int] = [] timings: list[float] = [] - for query in queries: + for index, query in enumerate(queries): + holder = as_holders[index] if as_holders is not None else None started = time.perf_counter() - hits = service.search(query, top_k=top_k).hits + hits = service.search(query, top_k=top_k, filters=SearchFilters(as_holder=holder)).hits timings.append(time.perf_counter() - started) # The product ranks atoms; collapse to sources by first (best) hit so the arms stay comparable. sources: list[str] = [] for hit in hits: - source_id = hit.atom.source_id + source_id = corpus_ids[hit.atom.source_id] if source_id not in sources: sources.append(source_id) - distinct.append(len({hit.atom.source_id for hit in hits})) + distinct.append(len({corpus_ids[hit.atom.source_id] for hit in hits})) rankings.append(sources[:top_k]) return rankings, distinct, timings diff --git a/bench/tests/test_joint.py b/bench/tests/test_joint.py index a674b4d..98ea385 100644 --- a/bench/tests/test_joint.py +++ b/bench/tests/test_joint.py @@ -181,7 +181,12 @@ def _report() -> dict: "live_product": { "queries": 2, "top_k": 10, + "recall_at_10": 0.87, + "mrr": 0.74, + "hit_at_10": 0.87, + "hit_at_10_wilson_lower": 0.82, "p50": 1.0, + "p95": 1.0, "mean": 1.0, "max": 1.0, "top10_overlap_with_bm25_only": 0.0, @@ -202,6 +207,7 @@ def test_the_report_writer_survives_a_scale_note(tmp_path: Path) -> None: assert "_latency only_" in text assert "load average 1.0 on 32 cores" in text assert "## unanswerable queries (50.0% of the suite)" in text + assert "recall@10 0.870" in text def test_scale_arm_names_reads_only_latency_dicts() -> None: