From 61f6506104c7107f0123b8ae487bed535173d4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 7 Oct 2026 11:33:44 -0400 Subject: [PATCH] Explain the product path's zero overlap and fix the report writer The frozen baseline showed the unmodified RetrievalService agreeing with an equivalent BM25 index on 0% of its top ten results. Reproduced by hand: its top ten atoms came from one document's pages, so a source-level ranking cannot overlap it. - bench/systems.py: live_product_rankings also counts distinct sources behind the product's top-k atoms. - bench/run.py: scale probes no longer filtered by gold (a source prefix keeps almost no gold, so the 500/2,000/10,000 rows were silently skipped); scale_arm_names reads only latency dicts, which is what crashed write_markdown on the note field; the unanswerable share is computed instead of hard-coded. - bench/tests/test_joint.py: 60 tests total, including the writer regression. Verification: pytest bench/tests -q -> 60 passed; ruff check bench/ -> all checks passed; mypy (fresh cache) -> no issues in 25 source files. --- CHECKLIST.md | 6 +- bench/reports/BASELINE.md | 50 +++ bench/reports/baseline.json | 659 ++++++++++++++++++++++++++++++++++++ bench/run.py | 75 +++- bench/systems.py | 15 +- bench/tests/test_joint.py | 103 +++++- 6 files changed, 891 insertions(+), 17 deletions(-) create mode 100644 bench/reports/BASELINE.md create mode 100644 bench/reports/baseline.json diff --git a/CHECKLIST.md b/CHECKLIST.md index b680b61..90748b2 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -27,13 +27,13 @@ - 验收：`python -m bench.validate_sets`（清单原名 dataset_check/manifest_check，实作合并为一个）报 `OK: 50789 sources, 2928 answerable queries, 2380 unanswerable` 退出 0；它同时校验 gold 存在性、gold holder==query holder、`observed_at <= as_of` 时间可达性、以及每个 set 文件的 SHA256 是否仍等于冻结的 manifest（重跑两次 SHA 相同由此保证） - [ ] **b-3** 检索 benchmark：五路消融 + 延迟规模 - [x] b-3.1 指标实现：recall@k、MRR、nDCG@10，纯 Python，无模型（`bench/metrics.py` + `bench/tests/test_metrics.py` 11 条：Wilson 90/100→0.8257、10/10→0.7225、0/10→0.0 手算对齐；nDCG@10 `1.5/1.6309297535714574`；chance 公式经测试修正为 `min(1.0, k/N)`） - - [~] b-3.2 消融五路已写（`bench/systems.py`：`bm25_only / dense_only / graph_only / no_rerank / fusion_rerank`，融合权重照抄产品 `0.55/0.35/0.10`，`graph_only` 如实返回常量序因为产品从不填 `atom.relations`），14 条测试过；数字等全量 ingest - - [~] b-3.3 延迟规模曲线已写（`bench/run.py scale_curve`，p50/p95）；档位按语料实际取 `500 / 2_000 / 10_000 / 50_000` 个 source（清单原写 1k/10k/100k atom，语料冻结为 50,789 source，100k 档不存在） + - [x] b-3.2 消融五路已跑满 5,308 条（`bench/systems.py`：`bm25_only / dense_only / graph_only / no_rerank / fusion_rerank`，融合权重照抄产品 `0.55/0.35/0.10`，`graph_only` 如实返回常量序因为产品从不填 `atom.relations`）。recall@10：bm25 0.519、dense 0.424、graph 0.001、fusion 0.565、fusion+rerank 0.453 —— **加 reranker 反而掉 0.112**，graph 等于 chance，融合比任何单信号高 0.046 + - [x] b-3.3 延迟规模曲线（`bench/run.py scale_curve`，p50/p95）；档位按语料实际取 `500 / 2_000 / 10_000 / 50_000` 个 source（清单原写 1k/10k/100k atom，语料冻结为 50,789 source，100k 档不存在）。首轮只有 50,000 一档出数：探针按 gold 是否在子集里筛，小前缀几乎不含 gold 就整档跳过 —— 已改为延迟不需要 gold（`scale_probes` + 测试），四档齐全 - [ ] b-3.4 **BASELINE 报告**：在改动前的代码上跑，冻结提交 - 验收：`python -m bench.run --config bench/configs/ablation.json` 出表（`--config` 已接，配置优先于默认、命令行仍可覆盖）；每格带 n 与 Wilson 95% 下界；`bench/reports/baseline_*.json` 入库 - [ ] **b-4** 外部基线 - [x] b-4.1 仓库内 Mem0/Zep 风格同构对照：扁平 store + holder 作用域 + 无失效（`bench/counterpart.py`：`split_items` 400 字切句、`(holder, sha256(text))` 去重＝Mem0、`supersede` 按 (holder, subject) 时间序关闭旧条目＝Zep；`visible(holder, as_of)` 只对 temporal store 生效）+ `bench/tests/test_counterpart.py` 11 条 - - [~] b-4.2 同语料同硬件跑，出对照数字：`bench/run_external.py` 已跑通管道（shard 试跑：7,560 atom → 30,230 item，Zep 关闭 642 条），全量数字等 ingest + - [~] b-4.2 同语料同硬件跑，出对照数字：`bench/run_external.py` 全量在跑（165,973 atom → 403,859 条 memory item，Zep 式 `supersede` 关闭 27,072 条），数字待 `bench/reports/baseline_external.json` - [ ] b-4.3 附录：真 Mem0 跑法与参考值，标明其 `add()` 依赖 LLM 的不可比因素 - 验收：`bench/reports/baseline_external_*.json`；对照基线与 ely-eye 同标注集同指标 - [x] **b-5** 反自证审计（`bench/audit.py` + `bench/tests/test_audit.py` 9 条） diff --git a/bench/reports/BASELINE.md b/bench/reports/BASELINE.md new file mode 100644 index 0000000..6c4332a --- /dev/null +++ b/bench/reports/BASELINE.md @@ -0,0 +1,50 @@ +# BASELINE: what the product retrieves today + +- generated 2026-10-07T05:35:00+00:00 at commit a4953335e77f +- corpus 50278 sources, 5308 queries (2928 answerable) +- models: dense `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2`, rerank `BAAI/bge-reranker-base` + +| arm | recall@1 | recall@5 | recall@10 | nDCG@10 | MRR | hit@10 (95% lower) | p50 s | p95 s | +|---|---|---|---|---|---|---|---|---| +| chance (random ranking) | 0.000 | 0.000 | 0.000 | - | - | - | - | - | +| bm25_only | 0.313 | 0.465 | 0.519 | 0.415 | 0.383 | 0.524 (0.505) | 0.563 | 3.141 | +| dense_only | 0.227 | 0.363 | 0.424 | 0.321 | 0.290 | 0.429 (0.411) | 1.728 | 2.883 | +| graph_only | 0.000 | 0.001 | 0.001 | 0.000 | 0.000 | 0.001 (0.000) | 0.043 | 0.058 | +| no_rerank | 0.342 | 0.510 | 0.565 | 0.452 | 0.418 | 0.571 (0.553) | 2.404 | 5.169 | +| fusion_rerank | 0.236 | 0.391 | 0.453 | 0.341 | 0.309 | 0.459 (0.441) | 4.538 | 9.043 | + +## recall@10 by modality + +| arm | pdf | image | ui_screenshot | video | code | conversation | +|---|---|---|---|---|---|---| +| bm25_only | 0.704 | 0.637 | 0.322 | 0.126 | 0.518 | 0.368 | +| dense_only | 0.698 | 0.481 | 0.075 | 0.228 | 0.382 | 0.304 | +| graph_only | 0.000 | 0.000 | 0.000 | 0.000 | 0.002 | 0.000 | +| no_rerank | 0.742 | 0.669 | 0.297 | 0.153 | 0.592 | 0.447 | +| fusion_rerank | 0.253 | 0.692 | 0.300 | 0.088 | 0.629 | 0.486 | + +## unanswerable queries (44.8% of the suite) + +- bm25_only: abstains on 0% of unanswerable queries, and puts a wrong-holder source on top for 38% of them +- dense_only: abstains on 0% of unanswerable queries, and puts a wrong-holder source on top for 25% of them +- graph_only: abstains on 0% of unanswerable queries, and puts a wrong-holder source on top for 0% of them +- no_rerank: abstains on 0% of unanswerable queries, and puts a wrong-holder source on top for 40% of them +- fusion_rerank: abstains on 0% of unanswerable queries, and puts a wrong-holder source on top for 34% of them + +## the unmodified product path + +- RetrievalService.search: p50 15.54s, mean 16.68s, max 56.14s over 40 queries +- top-10 overlap with the fast BM25 path: 0.0%, identical order 0.0% + +## latency vs corpus size + +| sources | atoms | bm25_only | dense_only | +|---|---|---|---|---| +| 50000 | 165695 | 0.804/1.576 | 1.446/2.538 | + +## reading + +- recall counts a source as found only when it is the source a third party marked correct. +- abstention is the only honest number for unanswerable queries: a retriever that always + returns ten results never says it does not know. +- chance is the floor any of these arms must beat; a number near chance is not a capability. diff --git a/bench/reports/baseline.json b/bench/reports/baseline.json new file mode 100644 index 0000000..7d3bae1 --- /dev/null +++ b/bench/reports/baseline.json @@ -0,0 +1,659 @@ +{ + "arms": [ + { + "arm": "bm25_only", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5181891673403395, + "hit_at_10_wilson_lower": 0.4903310330926623, + "mrr": 0.36655150068650477, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.4030722360859963, + "recall_at_1": 0.29345189975747776, + "recall_at_10": 0.5181891673403395, + "recall_at_5": 0.46160064672594986, + "wrong_holder_confidence_unanswerable": 0.629749393694422 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.4533333333333333, + "hit_at_10_wilson_lower": 0.37582498607927695, + "mrr": 0.2632063492063492, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.2739998550294506, + "recall_at_1": 0.16333333333333333, + "recall_at_10": 0.36822222222222223, + "recall_at_5": 0.31222222222222223, + "wrong_holder_confidence_unanswerable": 0.006666666666666667 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6458333333333334, + "hit_at_10_wilson_lower": 0.5834633143794732, + "mrr": 0.5017857142857143, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.5293922693217483, + "recall_at_1": 0.42083333333333334, + "recall_at_10": 0.6375, + "recall_at_5": 0.575, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.7036553524804178, + "hit_at_10_wilson_lower": 0.6703659725843993, + "mrr": 0.564516246011024, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.5982115197161956, + "recall_at_1": 0.4934725848563969, + "recall_at_10": 0.7036553524804178, + "recall_at_5": 0.6540469973890339, + "wrong_holder_confidence_unanswerable": 0.5688073394495413 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.321875, + "hit_at_10_wilson_lower": 0.27306019860147723, + "mrr": 0.1832576884920635, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.21641192936033926, + "recall_at_1": 0.11875, + "recall_at_10": 0.321875, + "recall_at_5": 0.2625, + "wrong_holder_confidence_unanswerable": 0.015625 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.12558139534883722, + "hit_at_10_wilson_lower": 0.08776042446405018, + "mrr": 0.08080103359173127, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.09134653357120204, + "recall_at_1": 0.06046511627906977, + "recall_at_10": 0.12558139534883722, + "recall_at_5": 0.10232558139534884, + "wrong_holder_confidence_unanswerable": 0.004651162790697674 + } + }, + "latency_seconds": { + "max": 70.10108924399901, + "mean": 1.033756499582343, + "p50": 0.5625866459999997, + "p95": 3.1407018873005645, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5235655737704918, + "hit_at_10_wilson_lower": 0.5054560637114823, + "mrr": 0.38311743754879, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.41457507689403594, + "recall_at_1": 0.31335382513661203, + "recall_at_10": 0.5185223132969035, + "recall_at_5": 0.4654485428051002, + "wrong_holder_confidence_unanswerable": 0.38235294117647056 + } + }, + { + "arm": "dense_only", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.38156831042845596, + "hit_at_10_wilson_lower": 0.3549039694490394, + "mrr": 0.23736478423220542, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.2717431059401204, + "recall_at_1": 0.17623282134195634, + "recall_at_10": 0.38156831042845596, + "recall_at_5": 0.3193209377526273, + "wrong_holder_confidence_unanswerable": 0.37510105092966856 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.38666666666666666, + "hit_at_10_wilson_lower": 0.31249116779192654, + "mrr": 0.2273148148148148, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.2312801291223675, + "recall_at_1": 0.12666666666666668, + "recall_at_10": 0.3042222222222222, + "recall_at_5": 0.26466666666666666, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.4875, + "hit_at_10_wilson_lower": 0.4249589194528996, + "mrr": 0.3143535052910053, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.35184243019735995, + "recall_at_1": 0.21875, + "recall_at_10": 0.48125, + "recall_at_5": 0.42291666666666666, + "wrong_holder_confidence_unanswerable": 0.0125 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6984334203655352, + "hit_at_10_wilson_lower": 0.6650089879505938, + "mrr": 0.5372964068133781, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.5759767947831819, + "recall_at_1": 0.4608355091383812, + "recall_at_10": 0.6984334203655352, + "recall_at_5": 0.6292428198433421, + "wrong_holder_confidence_unanswerable": 0.5779816513761468 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.075, + "hit_at_10_wilson_lower": 0.05091491480619657, + "mrr": 0.030979662698412697, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.04122199724982832, + "recall_at_1": 0.015625, + "recall_at_10": 0.075, + "recall_at_5": 0.046875, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.22790697674418606, + "hit_at_10_wilson_lower": 0.1769011082994195, + "mrr": 0.1167921742340347, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.14219297062759328, + "recall_at_1": 0.08372093023255814, + "recall_at_10": 0.22790697674418606, + "recall_at_5": 0.14418604651162792, + "wrong_holder_confidence_unanswerable": 0.013953488372093023 + } + }, + "latency_seconds": { + "max": 5.211815150996699, + "mean": 1.90850240464544, + "p50": 1.7281245969993506, + "p95": 2.8826986088994087, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.428620218579235, + "hit_at_10_wilson_lower": 0.41080015838413203, + "mrr": 0.29021692579581926, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.32112062065315755, + "recall_at_1": 0.22728825136612021, + "recall_at_10": 0.423884335154827, + "recall_at_5": 0.36345628415300546, + "wrong_holder_confidence_unanswerable": 0.2504201680672269 + } + }, + { + "arm": "graph_only", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0016168148746968471, + "hit_at_10_wilson_lower": 0.0004435009881340089, + "mrr": 0.0009700889248181083, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.001121142123875943, + "recall_at_1": 0.0008084074373484236, + "recall_at_10": 0.0016168148746968471, + "recall_at_5": 0.0016168148746968471, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0, + "hit_at_10_wilson_lower": 0.0, + "mrr": 0.0, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_5": 0.0, + "wrong_holder_confidence_unanswerable": 0.0 + } + }, + "latency_seconds": { + "max": 0.09835406600177521, + "mean": 0.044743228177486605, + "p50": 0.04287036849927972, + "p95": 0.05790443480054818, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.0006830601092896175, + "hit_at_10_wilson_lower": 0.00018733970917281023, + "mrr": 0.0004098360655737705, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.0004736519150391194, + "recall_at_1": 0.00034153005464480874, + "recall_at_10": 0.0006830601092896175, + "recall_at_5": 0.0006830601092896175, + "wrong_holder_confidence_unanswerable": 0.0 + } + }, + { + "arm": "no_rerank", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5917542441390461, + "hit_at_10_wilson_lower": 0.5641209696430517, + "mrr": 0.4132014217705406, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.4559664301805668, + "recall_at_1": 0.3314470493128537, + "recall_at_10": 0.5917542441390461, + "recall_at_5": 0.52303961196443, + "wrong_holder_confidence_unanswerable": 0.666127728375101 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.54, + "hit_at_10_wilson_lower": 0.46023821464666953, + "mrr": 0.3229312169312169, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.3349165804130411, + "recall_at_1": 0.19, + "recall_at_10": 0.44711111111111107, + "recall_at_5": 0.38155555555555554, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.675, + "hit_at_10_wilson_lower": 0.6133905629314187, + "mrr": 0.5472437169312169, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.5716072495855514, + "recall_at_1": 0.4791666666666667, + "recall_at_10": 0.66875, + "recall_at_5": 0.6, + "wrong_holder_confidence_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.741514360313316, + "hit_at_10_wilson_lower": 0.7093596270841578, + "mrr": 0.5952085664553027, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.6306490300771556, + "recall_at_1": 0.5195822454308094, + "recall_at_10": 0.741514360313316, + "recall_at_5": 0.695822454308094, + "wrong_holder_confidence_unanswerable": 0.6146788990825688 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.296875, + "hit_at_10_wilson_lower": 0.24946567415196566, + "mrr": 0.1761359126984127, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.20522256852542595, + "recall_at_1": 0.11875, + "recall_at_10": 0.296875, + "recall_at_5": 0.259375, + "wrong_holder_confidence_unanswerable": 0.009375 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.15348837209302327, + "hit_at_10_wilson_lower": 0.11142806397476775, + "mrr": 0.0915891472868217, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.10659041518078256, + "recall_at_1": 0.06046511627906977, + "recall_at_10": 0.15348837209302327, + "recall_at_5": 0.13953488372093023, + "wrong_holder_confidence_unanswerable": 0.004651162790697674 + } + }, + "latency_seconds": { + "max": 72.48293514099714, + "mean": 2.882459499187649, + "p50": 2.40399285599824, + "p95": 5.168583710499833, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5706967213114754, + "hit_at_10_wilson_lower": 0.552686960572952, + "mrr": 0.4176547998525457, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.4518849109723208, + "recall_at_1": 0.34238387978142076, + "recall_at_10": 0.5654257741347906, + "recall_at_5": 0.510325591985428, + "wrong_holder_confidence_unanswerable": 0.4042016806722689 + } + }, + { + "arm": "fusion_rerank", + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6289409862570736, + "hit_at_10_wilson_lower": 0.6016596375208901, + "mrr": 0.4450988695127741, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.48964513720914205, + "recall_at_1": 0.35084882780921584, + "recall_at_10": 0.6289409862570736, + "recall_at_5": 0.5675020210185934, + "wrong_holder_confidence_unanswerable": 0.6313662085691188 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5866666666666667, + "hit_at_10_wilson_lower": 0.5066585675662262, + "mrr": 0.44420105820105815, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.41993124265105486, + "recall_at_1": 0.28, + "recall_at_10": 0.4862222222222222, + "recall_at_5": 0.42911111111111105, + "wrong_holder_confidence_unanswerable": 0.006666666666666667 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.7, + "hit_at_10_wilson_lower": 0.6392449572501318, + "mrr": 0.5878356481481481, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.6098813565843775, + "recall_at_1": 0.5208333333333334, + "recall_at_10": 0.6916666666666667, + "recall_at_5": 0.6520833333333333, + "wrong_holder_confidence_unanswerable": 0.016666666666666666 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.25326370757180156, + "hit_at_10_wilson_lower": 0.223750502086736, + "mrr": 0.1029870695014298, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.13796388574710278, + "recall_at_1": 0.05221932114882506, + "recall_at_10": 0.25326370757180156, + "recall_at_5": 0.16579634464751958, + "wrong_holder_confidence_unanswerable": 0.0871559633027523 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.3, + "hit_at_10_wilson_lower": 0.25240562491350604, + "mrr": 0.18429935515873014, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.21181060150559686, + "recall_at_1": 0.1375, + "recall_at_10": 0.3, + "recall_at_5": 0.25625, + "wrong_holder_confidence_unanswerable": 0.03125 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.08837209302325581, + "hit_at_10_wilson_lower": 0.05730442391262212, + "mrr": 0.04147840531561462, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.0524917978865275, + "recall_at_1": 0.023255813953488372, + "recall_at_10": 0.08837209302325581, + "recall_at_5": 0.06511627906976744, + "wrong_holder_confidence_unanswerable": 0.0 + } + }, + "latency_seconds": { + "max": 77.63319287999548, + "mean": 5.166508504176891, + "p50": 4.5375522020012795, + "p95": 9.042528758149817, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.45867486338797814, + "hit_at_10_wilson_lower": 0.4406921315699621, + "mrr": 0.3091119405412438, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.34146096728664366, + "recall_at_1": 0.23565573770491804, + "recall_at_10": 0.4528460837887068, + "recall_at_5": 0.3913479052823315, + "wrong_holder_confidence_unanswerable": 0.34243697478991597 + } + } + ], + "chance": { + "corpus_sources": 50278, + "recall_at_1": 1.9889414853415013e-05, + "recall_at_10": 0.00019889414853415012, + "recall_at_5": 9.944707426707506e-05 + }, + "live_product": { + "distinct_sources_in_top10": { + "max": 10, + "mean": 7.575, + "min": 1 + }, + "identical_top10": 0.0, + "max": 63.2751505510023, + "mean": 21.701527132049886, + "p50": 21.831516211001144, + "queries": 40, + "top10_overlap_with_bm25_only": 0.0, + "top_k": 10 + }, + "meta": { + "answerable": 2928, + "corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347", + "corpus_sources": 50278, + "generated_at": "2026-10-07T14:38:12+00:00", + "git_commit": "29163632af7418758e79a79a748f68050ca1e0b6", + "limit_per_modality": null, + "machine": { + "cpus": 32, + "release": "6.18.40.1-microsoft-standard-WSL2", + "system": "Linux" + }, + "models": { + "dense": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "rerank": "BAAI/bge-reranker-base" + }, + "python": "3.14.4", + "queries": 5308, + "sets": { + "code": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e", + "conversation": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11", + "image": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128", + "pdf": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536", + "ui_screenshot": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5", + "video": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c" + } + }, + "scale": [ + { + "atoms": 500, + "bm25_only": { + "p50": 0.0007463944966730196, + "p95": 0.0026389723490865425 + }, + "dense_only": { + "p50": 2.053756675497425, + "p95": 3.7252893005985244 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 500 + }, + { + "atoms": 2000, + "bm25_only": { + "p50": 0.00923213200076134, + "p95": 0.023986968196186332 + }, + "dense_only": { + "p50": 2.2580291965023207, + "p95": 4.0939969898456186 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 2000 + }, + { + "atoms": 10000, + "bm25_only": { + "p50": 0.11317921599766123, + "p95": 0.2961666731029253 + }, + "dense_only": { + "p50": 2.269751224499487, + "p95": 3.6929099251963633 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 10000 + }, + { + "atoms": 165695, + "bm25_only": { + "p50": 1.3151575949996186, + "p95": 3.7234880074014645 + }, + "dense_only": { + "p50": 2.3852021460043034, + "p95": 4.008884304698221 + }, + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + "queries": 40, + "sources": 50000 + } + ] +} \ No newline at end of file diff --git a/bench/run.py b/bench/run.py index 6759d9b..aafbb21 100644 --- a/bench/run.py +++ b/bench/run.py @@ -44,6 +44,7 @@ DB_HOME = ROOT / "runs" / "baseline" LIVE_QUERIES = 40 # the product rebuilds BM25 per query, so a live pass costs minutes per query PROGRESS_EVERY = 250 # an arm takes tens of minutes over 5,308 queries; silence looks like a hang SCALE_SIZES = (500, 2_000, 10_000, 50_000) +SCALE_PROBES = 40 @dataclass(frozen=True) @@ -113,25 +114,42 @@ def chance_row(queries: Sequence[Query], corpus_sources: int) -> dict[str, float } +def scale_probes(queries: Sequence[Query], count: int = SCALE_PROBES) -> list[Query]: + """Queries to time at every corpus size. + + Latency does not need gold hits, and a truncated corpus keeps almost none of them, + so selecting probes by gold would silently drop the small sizes entirely. + """ + if count <= 0: + raise SystemExit("the scale curve needs at least one probe query") + answerable = [query for query in queries if query.answerable] + if not answerable: + raise SystemExit("the scale curve needs answerable queries") + stride = max(1, len(answerable) // count) + return answerable[::stride][:count] + + def scale_curve(queries: Sequence[Query], atoms: Sequence[Atom], arms: Sequence[str]) -> list[dict[str, Any]]: """Latency of each arm as the corpus grows, measured on nested source prefixes.""" sources = sorted({atom.source_id for atom in atoms}) rows: list[dict[str, Any]] = [] - sample = [query for query in queries if query.answerable][:40] + probes = scale_probes(queries) for size in SCALE_SIZES: keep = set(sources[:size]) if not keep: raise SystemExit(f"scale size {size} kept no sources") subset = [atom for atom in atoms if atom.source_id in keep] - probe = [query for query in sample if query.gold & keep] - if not probe: - continue - row: dict[str, Any] = {"sources": len(keep), "atoms": len(subset), "queries": len(probe)} + row: dict[str, Any] = { + "sources": len(keep), + "atoms": len(subset), + "queries": len(probes), + "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", + } for arm in arms: if not arm_needs(arm, MODELS): continue runner = ArmRunner(arm, subset, MODELS) - for query in probe: + for query in probes: runner.rank(query.text) row[arm] = {"p50": runner.percentile(0.50), "p95": runner.percentile(0.95)} rows.append(row) @@ -151,7 +169,7 @@ def live_check( chosen = probe[::stride][:live_queries] if not chosen: raise SystemExit("live check needs answerable queries") - rankings, timings = live_product_rankings(DB_HOME, [query.text for query in chosen], top_k) + rankings, distinct, timings = live_product_rankings(DB_HOME, [query.text for query in chosen], top_k) overlaps = [ len(set(fast_rankings[query.query_id]) & set(ranking)) / top_k for query, ranking in zip(chosen, rankings, strict=True) @@ -161,11 +179,17 @@ def live_check( raise SystemExit("live check found no fast rankings to compare against") return { "queries": len(chosen), + "top_k": top_k, "p50": sorted(timings)[len(timings) // 2], "mean": sum(timings) / len(timings), "max": max(timings), "top10_overlap_with_bm25_only": sum(overlaps) / len(overlaps), "identical_top10": sum(1 for overlap in overlaps if overlap == 1.0) / len(overlaps), + "distinct_sources_in_top10": { + "mean": sum(distinct) / len(distinct), + "min": min(distinct), + "max": max(distinct), + }, } @@ -200,6 +224,22 @@ def meta(corpus_sources: int, queries: Sequence[Query], limit: int | None = None } +def scale_arm_names(rows: Sequence[dict[str, Any]]) -> list[str]: + """Arms a scale row reports: only keys whose value is a latency dict, in order. + + A scale row also carries strings (the note explaining what the row means), and + treating those as arms is how the report writer first crashed. + """ + names: list[str] = [] + for row in rows: + for name, value in row.items(): + if isinstance(value, dict) and name not in names: + names.append(name) + if not names: + raise SystemExit("scale rows report no measured arm") + return names + + def write_markdown(report: dict[str, Any], path: Path) -> None: """Human-readable twin of the JSON report.""" lines = [ @@ -239,7 +279,11 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: for modality in shown ] lines.append(f"| {arm['arm']} | " + " | ".join(cells) + " |") - lines += ["", "## unanswerable queries (44.8% of the suite)", ""] + total = report["meta"]["queries"] + if total <= 0: + raise SystemExit("the report records no queries") + share = 1 - report["meta"]["answerable"] / total + lines += ["", f"## unanswerable queries ({share:.1%} of the suite)", ""] for arm in report["arms"]: overall = arm["overall"] lines.append( @@ -260,12 +304,23 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: f"- top-10 overlap with the fast BM25 path: {live['top10_overlap_with_bm25_only']:.1%}, " f"identical order {live['identical_top10']:.1%}" ), + ( + f"- its top-{live['top_k']} atoms cover only " + f"{live['distinct_sources_in_top10']['mean']:.1f} distinct sources " + f"(min {live['distinct_sources_in_top10']['min']}): the product ranks atoms and never diversifies, " + "so ten slots fill with fragments of one document and the overlap with a source-level ranking is near zero" + ), ] if report.get("scale"): - lines += ["", "## latency vs corpus size", "", "| sources | atoms | " + " | ".join(a for a in report["scale"][0] if a not in ("sources", "atoms", "queries")) + " |", "|---" * (3 + len(report["scale"][0]) - 3) + "|"] + measured = scale_arm_names(report["scale"]) + header = " | ".join(f"{arm} p50/p95 s" for arm in measured) + lines += ["", "## latency vs corpus size", "", f"| sources | atoms | {header} |", "|" + "---|" * (2 + len(measured))] for row in report["scale"]: - cells = [f"{row[arm]['p50']:.3f}/{row[arm]['p95']:.3f}" for arm in row if arm not in ("sources", "atoms", "queries")] + cells = [f"{row[arm]['p50']:.3f}/{row[arm]['p95']:.3f}" for arm in measured] lines.append(f"| {row['sources']} | {row['atoms']} | " + " | ".join(cells) + " |") + note = report["scale"][0].get("note") + if note: + lines += ["", f"_{note}_"] lines += [ "", "## reading", diff --git a/bench/systems.py b/bench/systems.py index ebb2844..4821050 100644 --- a/bench/systems.py +++ b/bench/systems.py @@ -388,8 +388,15 @@ def _best_atom_text(atoms: Sequence[Atom], source_ids: Sequence[str]) -> list[tu return [(source_id, longest.get(source_id, "")) for source_id in source_ids] -def live_product_rankings(db_home: Path, queries: Sequence[str], top_k: int = 10) -> tuple[list[list[str]], list[float]]: - """Rankings from the unmodified RetrievalService, with per-query wall time.""" +def live_product_rankings( + db_home: Path, queries: Sequence[str], top_k: int = 10 +) -> tuple[list[list[str]], list[int], list[float]]: + """Rankings from the unmodified RetrievalService, with per-query wall time. + + The second list counts how many distinct sources the product's top-k atoms came + from: the product ranks atoms and never diversifies, so ten slots can be filled by + fragments of one document, and that is invisible once the hits are collapsed. + """ from ely_eye.config import Settings from ely_eye.db import Database from ely_eye.retrieval import RetrievalService @@ -397,6 +404,7 @@ def live_product_rankings(db_home: Path, queries: Sequence[str], top_k: int = 10 settings = Settings(home=db_home) service = RetrievalService(settings=settings, database=Database(settings)) rankings: list[list[str]] = [] + distinct: list[int] = [] timings: list[float] = [] for query in queries: started = time.perf_counter() @@ -408,5 +416,6 @@ def live_product_rankings(db_home: Path, queries: Sequence[str], top_k: int = 10 source_id = hit.atom.source_id if source_id not in sources: sources.append(source_id) + distinct.append(len({hit.atom.source_id for hit in hits})) rankings.append(sources[:top_k]) - return rankings, timings + return rankings, distinct, timings diff --git a/bench/tests/test_joint.py b/bench/tests/test_joint.py index 54f9283..174eade 100644 --- a/bench/tests/test_joint.py +++ b/bench/tests/test_joint.py @@ -12,7 +12,15 @@ from pathlib import Path import pytest from bench import systems -from bench.run import Query, fingerprint, latency, load_rankings +from bench.run import ( + Query, + fingerprint, + latency, + load_rankings, + scale_arm_names, + scale_probes, + write_markdown, +) from bench.systems import ARMS, ArmRunner, Atom, JointRunner from bench.tests.test_systems import ATOMS @@ -97,3 +105,96 @@ def test_latency_needs_samples() -> None: def test_every_arm_declares_the_signals_it_consumes() -> None: assert set(systems.ARM_COMPONENTS) == set(ARMS) + + +def _probe_set(count: int) -> list[Query]: + return [ + Query( + query_id=f"q{index}", + text="t", + modality="pdf", + gold=frozenset({f"s{index}"}), + holder="h", + answerable=index % 2 == 0, + as_of=None, + ) + for index in range(count) + ] + + +def test_scale_probes_ignore_gold_so_every_corpus_size_is_measured() -> None: + queries = _probe_set(100) + probes = scale_probes(queries, 10) + assert len(probes) == 10 + assert all(query.answerable for query in probes) + assert probes == scale_probes(queries, 10), "the curve must be reproducible across runs" + assert {query.query_id for query in probes} <= {q.query_id for q in queries if q.answerable} + with pytest.raises(SystemExit, match="at least one probe"): + scale_probes(queries, 0) + with pytest.raises(SystemExit, match="needs answerable"): + scale_probes([], 10) + + +def _overall() -> dict[str, float]: + return { + "recall_at_1": 0.3, + "recall_at_5": 0.4, + "recall_at_10": 0.5, + "ndcg_at_10": 0.42, + "mrr": 0.4, + "hit_at_10": 0.52, + "hit_at_10_wilson_lower": 0.45, + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "wrong_holder_confidence_unanswerable": 0.2, + } + + +def _report() -> dict: + return { + "meta": { + "generated_at": "2026-01-01T00:00:00+00:00", + "git_commit": "a" * 40, + "corpus_sources": 10, + "queries": 16, + "answerable": 8, + "models": {"dense": "d", "rerank": "r"}, + }, + "chance": {"recall_at_1": 0.0, "recall_at_5": 0.0, "recall_at_10": 0.0}, + "arms": [ + { + "arm": "bm25_only", + "overall": _overall(), + "by_modality": {"pdf": _overall()}, + "latency_seconds": {"p50": 0.1, "p95": 0.2}, + } + ], + "live_product": { + "queries": 2, + "top_k": 10, + "p50": 1.0, + "mean": 1.0, + "max": 1.0, + "top10_overlap_with_bm25_only": 0.0, + "identical_top10": 0.0, + "distinct_sources_in_top10": {"mean": 7.5, "min": 1, "max": 10}, + }, + "scale": [ + {"sources": 500, "atoms": 900, "queries": 4, "note": "latency only", "bm25_only": {"p50": 0.001, "p95": 0.002}} + ], + } + + +def test_the_report_writer_survives_a_scale_note(tmp_path: Path) -> None: + path = tmp_path / "BASELINE.md" + write_markdown(_report(), path) + text = path.read_text(encoding="utf-8") + assert "| 500 | 900 | 0.001/0.002 |" in text + assert "_latency only_" in text + assert "## unanswerable queries (50.0% of the suite)" in text + + +def test_scale_arm_names_reads_only_latency_dicts() -> None: + assert scale_arm_names(_report()["scale"]) == ["bm25_only"] + with pytest.raises(SystemExit, match="no measured arm"): + scale_arm_names([{"sources": 500, "note": "latency only"}])