ZacharyZhang-NY/ely-eye
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
ely-eye/bench/reports/baseline_external.json
T
ZacharyZhang-NY 6de7439db6 Score the product path against corpus ids, not storage ids 
The frozen baseline claimed the product's top-10 shared 0.0% with the fast
BM25 path. That was a harness bug: gold names a corpus source
(mmlongbench:PH_2016.06.08_Economy-Final.pdf) while the store names a content
hash (PH_2016_06_08_Economy_Final_10e57152f44d1a2b), so the two id spaces never
met and the right document, ranked first, scored as a miss.

- bench/systems.py: `_corpus_ids` joins store sources through the
  `source_map()` the counterpart already uses, and refuses a store whose
  sources are not corpus files instead of silently scoring zero.
- bench/systems.py: `live_product_rankings(..., as_holders=...)` asks the
  product as the query's own holder, which is how the counterpart is scored.
- bench/run.py: the live section now reports recall@10 / MRR / hit@10 with a
  95% lower bound, and probes 300 queries instead of 40 — the SQL path answers
  in well under a second, so the old cap was a cost estimate that expired.
- bench/run.py: `--name CURRENT` writes a post-P0 report beside the frozen
  BASELINE without touching it, stamped with the commit and the store schema
  version the pass started on (the live pass migrates the store mid-run, so a
  plain stamp would be a lie).
- bench/stamp_holders.py: the pre-P0 store holds 165,973 memories nobody owns.
  The corpus index states the owner, so holders, scenarios and event times are
  stamped in place — 165,973 atoms across 50,278 sources, 0 left unowned.

Numbers, same machine, load 1.0-1.7:
- CURRENT (post-P0, schema v5): no_rerank recall@10 0.565 p50 0.643s;
  fusion+rerank 0.453 p50 7.000s; graph 0.001.
- Product as its holder: recall@10 0.870, MRR 0.740, n=173.
- Counterpart: mem0_flat holder scope 0.866, whole corpus 0.412; the holder
  boundary is worth +0.454 with an identical ranker and identical items.

Verification: pytest backend/tests bench/tests -q -> 173 passed; ruff and mypy
(fresh cache) clean on every touched file.
2026-10-07 19:32:32 -04:00

487 lines
15 KiB
JSON
Raw Permalink Blame History

{
"chance": {
"median_holder_sources": 43.0,
"note": "expected recall@10 of a random ranking inside the candidate set each row is allowed to see",
"recall_at_10": 0.00019689302801787788,
"recall_at_10_holder_scope": 0.3685528769188164
},
"meta": {
"answerable": 2928,
"extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator",
"generated_at": "2026-10-07T19:16:08+00:00",
"git_commit": "3d792417cfd0554cbe2bae0cfc098bb5bb07e28c",
"limit_per_modality": null,
"machine": {
"cpus": 32,
"load_average_1_5_15": [
2.85,
10.0,
14.84
],
"release": "6.18.40.1-microsoft-standard-WSL2",
"system": "Linux",
"threads_per_query": 4
},
"queries": 5308,
"queries_by_modality": {
"code": 2474,
"conversation": 300,
"image": 480,
"pdf": 984,
"ui_screenshot": 640,
"video": 430
},
"sources": 50789,
"top_k": 10
},
"stores": [
{
"by_modality": {
"code": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.952303961196443,
"hit_at_10_wilson_lower": 0.938963104367135,
"mrr": 0.6949699092787209,
"n_answerable": 1237,
"n_unanswerable": 1237,
"ndcg_at_10": 0.7581529489950144,
"recall_at_1": 0.549717057396928,
"recall_at_10": 0.952303961196443,
"recall_at_5": 0.8884397736459175,
"top_hit_from_holder_unanswerable": 1.0
},
"conversation": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 1.0,
"hit_at_10_wilson_lower": 0.9750297556319234,
"mrr": 0.9271111111111111,
"n_answerable": 150,
"n_unanswerable": 150,
"ndcg_at_10": 0.9366637917237846,
"recall_at_1": 0.657888888888889,
"recall_at_10": 0.9927777777777778,
"recall_at_5": 0.9814444444444445,
"top_hit_from_holder_unanswerable": 1.0
},
"image": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.6541666666666667,
"hit_at_10_wilson_lower": 0.5919888820581466,
"mrr": 0.504917328042328,
"n_answerable": 240,
"n_unanswerable": 240,
"ndcg_at_10": 0.5355835725491713,
"recall_at_1": 0.42291666666666666,
"recall_at_10": 0.6458333333333334,
"recall_at_5": 0.6,
"top_hit_from_holder_unanswerable": 1.0
},
"pdf": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.9164490861618799,
"hit_at_10_wilson_lower": 0.8947140126781276,
"mrr": 0.691579634464752,
"n_answerable": 766,
"n_unanswerable": 218,
"ndcg_at_10": 0.7454292373597072,
"recall_at_1": 0.587467362924282,
"recall_at_10": 0.9164490861618799,
"recall_at_5": 0.825065274151436,
"top_hit_from_holder_unanswerable": 1.0
},
"ui_screenshot": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.621875,
"hit_at_10_wilson_lower": 0.567595266832662,
"mrr": 0.32982886904761904,
"n_answerable": 320,
"n_unanswerable": 320,
"ndcg_at_10": 0.3983431061821887,
"recall_at_1": 0.21875,
"recall_at_10": 0.621875,
"recall_at_5": 0.465625,
"top_hit_from_holder_unanswerable": 1.0
},
"video": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.7162790697674418,
"hit_at_10_wilson_lower": 0.6526351336615988,
"mrr": 0.3956736803248431,
"n_answerable": 215,
"n_unanswerable": 215,
"ndcg_at_10": 0.47027044859447076,
"recall_at_1": 0.2837209302325581,
"recall_at_10": 0.7162790697674418,
"recall_at_5": 0.5255813953488372,
"top_hit_from_holder_unanswerable": 1.0
}
},
"closed_items": 0,
"items": 403859,
"latency_seconds": {
"max": 5.168096734996652,
"mean": 0.08254052409894046,
"p50": 0.08343072250136174,
"p95": 0.10547906984866127,
"queries": 5308
},
"overall": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.8674863387978142,
"hit_at_10_wilson_lower": 0.854722713774439,
"mrr": 0.6285142358400555,
"n_answerable": 2928,
"n_unanswerable": 2380,
"ndcg_at_10": 0.6852635110385413,
"recall_at_1": 0.49903802367941713,
"recall_at_10": 0.8664332877959927,
"recall_at_5": 0.7801286429872496,
"top_hit_from_holder_unanswerable": 1.0
},
"store": "mem0_flat [holder scope]"
},
{
"by_modality": {
"code": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.349232012934519,
"hit_at_10_wilson_lower": 0.3231693767503265,
"mrr": 0.21579377911229164,
"n_answerable": 1237,
"n_unanswerable": 1237,
"ndcg_at_10": 0.24771932445387607,
"recall_at_1": 0.15683104284559418,
"recall_at_10": 0.349232012934519,
"recall_at_5": 0.2991107518189167,
"top_hit_from_holder_unanswerable": 0.0
},
"conversation": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.5733333333333334,
"hit_at_10_wilson_lower": 0.4933252687249978,
"mrr": 0.37327513227513226,
"n_answerable": 150,
"n_unanswerable": 150,
"ndcg_at_10": 0.38064109675098423,
"recall_at_1": 0.2188888888888889,
"recall_at_10": 0.5004444444444445,
"recall_at_5": 0.4146666666666667,
"top_hit_from_holder_unanswerable": 0.0
},
"image": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.5333333333333333,
"hit_at_10_wilson_lower": 0.4701883065323697,
"mrr": 0.36166170634920636,
"n_answerable": 240,
"n_unanswerable": 240,
"ndcg_at_10": 0.3979762690192957,
"recall_at_1": 0.26458333333333334,
"recall_at_10": 0.525,
"recall_at_5": 0.4708333333333333,
"top_hit_from_holder_unanswerable": 0.0
},
"pdf": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.6827676240208878,
"hit_at_10_wilson_lower": 0.6489674562308053,
"mrr": 0.5150985328857391,
"n_answerable": 766,
"n_unanswerable": 218,
"ndcg_at_10": 0.555124937448977,
"recall_at_1": 0.43994778067885115,
"recall_at_10": 0.6827676240208878,
"recall_at_5": 0.6109660574412533,
"top_hit_from_holder_unanswerable": 0.0
},
"ui_screenshot": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.075,
"hit_at_10_wilson_lower": 0.05091491480619657,
"mrr": 0.032880704365079366,
"n_answerable": 320,
"n_unanswerable": 320,
"ndcg_at_10": 0.04277852579814079,
"recall_at_1": 0.01875,
"recall_at_10": 0.075,
"recall_at_5": 0.059375,
"top_hit_from_holder_unanswerable": 0.0
},
"video": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.11627906976744186,
"hit_at_10_wilson_lower": 0.08001303523895324,
"mrr": 0.06852713178294573,
"n_answerable": 215,
"n_unanswerable": 215,
"ndcg_at_10": 0.08038135054972745,
"recall_at_1": 0.04186046511627907,
"recall_at_10": 0.11627906976744186,
"recall_at_5": 0.11627906976744186,
"top_hit_from_holder_unanswerable": 0.0
}
},
"closed_items": 0,
"items": 403859,
"latency_seconds": {
"max": 0.1391175699973246,
"mean": 0.08744079960348271,
"p50": 0.0859174314973643,
"p95": 0.11227948825326166,
"queries": 5308
},
"overall": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.41598360655737704,
"hit_at_10_wilson_lower": 0.39825196843019733,
"mrr": 0.2833154436638043,
"n_answerable": 2928,
"n_unanswerable": 2380,
"ndcg_at_10": 0.31258063324325003,
"recall_at_1": 0.2193761384335155,
"recall_at_10": 0.41156648451730415,
"recall_at_5": 0.3610655737704918,
"top_hit_from_holder_unanswerable": 0.0
},
"store": "mem0_flat [whole corpus]"
},
{
"by_modality": {
"code": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.952303961196443,
"hit_at_10_wilson_lower": 0.938963104367135,
"mrr": 0.6949699092787209,
"n_answerable": 1237,
"n_unanswerable": 1237,
"ndcg_at_10": 0.7581529489950144,
"recall_at_1": 0.549717057396928,
"recall_at_10": 0.952303961196443,
"recall_at_5": 0.8884397736459175,
"top_hit_from_holder_unanswerable": 1.0
},
"conversation": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.11333333333333333,
"hit_at_10": 0.9866666666666667,
"hit_at_10_wilson_lower": 0.9526930912996325,
"mrr": 0.9053968253968253,
"n_answerable": 150,
"n_unanswerable": 150,
"ndcg_at_10": 0.9136591750866607,
"recall_at_1": 0.637888888888889,
"recall_at_10": 0.9783333333333334,
"recall_at_5": 0.9458888888888889,
"top_hit_from_holder_unanswerable": 0.8866666666666667
},
"image": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.5,
"hit_at_10": 0.6541666666666667,
"hit_at_10_wilson_lower": 0.5919888820581466,
"mrr": 0.504917328042328,
"n_answerable": 240,
"n_unanswerable": 240,
"ndcg_at_10": 0.5355835725491713,
"recall_at_1": 0.42291666666666666,
"recall_at_10": 0.6458333333333334,
"recall_at_5": 0.6,
"top_hit_from_holder_unanswerable": 0.5
},
"pdf": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.9164490861618799,
"hit_at_10_wilson_lower": 0.8947140126781276,
"mrr": 0.691579634464752,
"n_answerable": 766,
"n_unanswerable": 218,
"ndcg_at_10": 0.7454292373597072,
"recall_at_1": 0.587467362924282,
"recall_at_10": 0.9164490861618799,
"recall_at_5": 0.825065274151436,
"top_hit_from_holder_unanswerable": 1.0
},
"ui_screenshot": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.621875,
"hit_at_10_wilson_lower": 0.567595266832662,
"mrr": 0.32982886904761904,
"n_answerable": 320,
"n_unanswerable": 320,
"ndcg_at_10": 0.3983431061821887,
"recall_at_1": 0.21875,
"recall_at_10": 0.621875,
"recall_at_5": 0.465625,
"top_hit_from_holder_unanswerable": 1.0
},
"video": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.7162790697674418,
"hit_at_10_wilson_lower": 0.6526351336615988,
"mrr": 0.3956736803248431,
"n_answerable": 215,
"n_unanswerable": 215,
"ndcg_at_10": 0.47027044859447076,
"recall_at_1": 0.2837209302325581,
"recall_at_10": 0.7162790697674418,
"recall_at_5": 0.5255813953488372,
"top_hit_from_holder_unanswerable": 1.0
}
},
"closed_items": 27072,
"items": 403859,
"latency_seconds": {
"max": 1.9491578790039057,
"mean": 0.08603176019042821,
"p50": 0.0841971809968527,
"p95": 0.11250105440230979,
"queries": 5308
},
"overall": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.05756302521008403,
"hit_at_10": 0.8668032786885246,
"hit_at_10_wilson_lower": 0.8540138593279664,
"mrr": 0.6274018236620695,
"n_answerable": 2928,
"n_unanswerable": 2380,
"ndcg_at_10": 0.6840849958419674,
"recall_at_1": 0.4980134335154827,
"recall_at_10": 0.865693306010929,
"recall_at_5": 0.7783071493624771,
"top_hit_from_holder_unanswerable": 0.942436974789916
},
"store": "zep_temporal [holder scope]"
},
{
"by_modality": {
"code": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.35004042037186744,
"hit_at_10_wilson_lower": 0.32396114426502914,
"mrr": 0.21595449820995496,
"n_answerable": 1237,
"n_unanswerable": 1237,
"ndcg_at_10": 0.24802826744611478,
"recall_at_1": 0.15683104284559418,
"recall_at_10": 0.35004042037186744,
"recall_at_5": 0.2991107518189167,
"top_hit_from_holder_unanswerable": 0.0
},
"conversation": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.6666666666666666,
"hit_at_10_wilson_lower": 0.5878975668678504,
"mrr": 0.42928042328042326,
"n_answerable": 150,
"n_unanswerable": 150,
"ndcg_at_10": 0.4447119968977239,
"recall_at_1": 0.2391111111111111,
"recall_at_10": 0.5912222222222222,
"recall_at_5": 0.4597777777777778,
"top_hit_from_holder_unanswerable": 0.0
},
"image": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.5708333333333333,
"hit_at_10_wilson_lower": 0.5075830430993526,
"mrr": 0.4262020502645502,
"n_answerable": 240,
"n_unanswerable": 240,
"ndcg_at_10": 0.45449292583858186,
"recall_at_1": 0.35208333333333336,
"recall_at_10": 0.5625,
"recall_at_5": 0.5083333333333333,
"top_hit_from_holder_unanswerable": 0.0
},
"pdf": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.685378590078329,
"hit_at_10_wilson_lower": 0.6516380300993592,
"mrr": 0.518995296116706,
"n_answerable": 766,
"n_unanswerable": 218,
"ndcg_at_10": 0.5587707901607282,
"recall_at_1": 0.44386422976501305,
"recall_at_10": 0.685378590078329,
"recall_at_5": 0.6174934725848564,
"top_hit_from_holder_unanswerable": 0.0
},
"ui_screenshot": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.11875,
"hit_at_10_wilson_lower": 0.08775049817160425,
"mrr": 0.05682539682539682,
"n_answerable": 320,
"n_unanswerable": 320,
"ndcg_at_10": 0.0711722029605294,
"recall_at_1": 0.0375,
"recall_at_10": 0.11875,
"recall_at_5": 0.08125,
"top_hit_from_holder_unanswerable": 0.0
},
"video": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.16279069767441862,
"hit_at_10_wilson_lower": 0.1194411092694439,
"mrr": 0.08660022148394242,
"n_answerable": 215,
"n_unanswerable": 215,
"ndcg_at_10": 0.10455353510193785,
"recall_at_1": 0.05581395348837209,
"recall_at_10": 0.16279069767441862,
"recall_at_5": 0.12093023255813953,
"top_hit_from_holder_unanswerable": 0.0
}
},
"closed_items": 27072,
"items": 403859,
"latency_seconds": {
"max": 0.17060414699517423,
"mean": 0.0951456186994907,
"p50": 0.09420693750143982,
"p95": 0.12111507409863406,
"queries": 5308
},
"overall": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.43306010928961747,
"hit_at_10_wilson_lower": 0.41521181569494575,
"mrr": 0.29650609332986383,
"n_answerable": 2928,
"n_unanswerable": 2380,
"ndcg_at_10": 0.32645785820497475,
"recall_at_1": 0.23168260473588342,
"recall_at_10": 0.42851206739526415,
"recall_at_5": 0.3708902550091075,
"top_hit_from_holder_unanswerable": 0.0
},
"store": "zep_temporal [whole corpus]"
}
]
}
Reference in new issue View Git Blame Copy Permalink