ZacharyZhang-NY/ely-eye
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
dbe1d2cc82f18658bf5b0e081dd34c0a5bd3c52a
ely-eye/bench/reports/baseline_external.json
T
ZacharyZhang-NY dbe1d2cc82 Measure the counterpart honestly: warm models, vectorised search, scope split 
Three defects made the external baseline numbers and labels wrong.

- bench/systems.py: _sentence_model rebuilt the model on every query, so dense arm
  latency was mostly model loading and the latency-vs-size curve was flat because a
  constant dominated it. Models are now cached per process.
- bench/counterpart.py: visible() and search() walked 404k Python objects per query.
  Per-item holder/source/time columns are built once and the query path is numpy;
  12.5 queries/s instead of 0.7.
- bench/metrics.py: the aggregate key was named wrong_holder_confidence_unanswerable
  while it counts top hits that ARE from the asking holder. BASELINE.md read that
  backwards. Renamed to top_hit_from_holder_unanswerable and the sentence rewritten.
- bench/run_external.py: the by-modality table read row['answerable'] but aggregate
  emits n_answerable, so the writer died after scoring finished. Each store is now
  scored twice, [holder scope] and [whole corpus], so the gain from the holder
  boundary is separable from the gain from ranking; progress lines every 500 queries.

Verification: pytest bench/tests -q -> 61 passed; ruff check bench/ -> all checks
passed; mypy (fresh cache) -> no issues in 25 source files.
2026-10-07 13:33:13 -04:00

256 lines
7.8 KiB
JSON
Raw Blame History

{
"meta": {
"answerable": 2928,
"extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator",
"generated_at": "2026-10-07T16:54:23+00:00",
"git_commit": "c4e2982506faaf350a4b06738923899fc44bdedd",
"limit_per_modality": null,
"machine": {
"cpus": 32,
"load_average_1_5_15": [
6.23,
4.72,
4.44
],
"release": "6.18.40.1-microsoft-standard-WSL2",
"system": "Linux"
},
"queries": 5308,
"queries_by_modality": {
"code": 2474,
"conversation": 300,
"image": 480,
"pdf": 984,
"ui_screenshot": 640,
"video": 430
},
"sources": 50789,
"top_k": 10
},
"stores": [
{
"by_modality": {
"code": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.952303961196443,
"hit_at_10_wilson_lower": 0.938963104367135,
"mrr": 0.6949699092787209,
"n_answerable": 1237,
"n_unanswerable": 1237,
"ndcg_at_10": 0.7581529489950144,
"recall_at_1": 0.549717057396928,
"recall_at_10": 0.952303961196443,
"recall_at_5": 0.8884397736459175,
"wrong_holder_confidence_unanswerable": 1.0
},
"conversation": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 1.0,
"hit_at_10_wilson_lower": 0.9750297556319234,
"mrr": 0.9271111111111111,
"n_answerable": 150,
"n_unanswerable": 150,
"ndcg_at_10": 0.9366637917237846,
"recall_at_1": 0.657888888888889,
"recall_at_10": 0.9927777777777778,
"recall_at_5": 0.9814444444444445,
"wrong_holder_confidence_unanswerable": 1.0
},
"image": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.6541666666666667,
"hit_at_10_wilson_lower": 0.5919888820581466,
"mrr": 0.504917328042328,
"n_answerable": 240,
"n_unanswerable": 240,
"ndcg_at_10": 0.5355835725491713,
"recall_at_1": 0.42291666666666666,
"recall_at_10": 0.6458333333333334,
"recall_at_5": 0.6,
"wrong_holder_confidence_unanswerable": 1.0
},
"pdf": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.9164490861618799,
"hit_at_10_wilson_lower": 0.8947140126781276,
"mrr": 0.691579634464752,
"n_answerable": 766,
"n_unanswerable": 218,
"ndcg_at_10": 0.7454292373597072,
"recall_at_1": 0.587467362924282,
"recall_at_10": 0.9164490861618799,
"recall_at_5": 0.825065274151436,
"wrong_holder_confidence_unanswerable": 1.0
},
"ui_screenshot": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.621875,
"hit_at_10_wilson_lower": 0.567595266832662,
"mrr": 0.32982886904761904,
"n_answerable": 320,
"n_unanswerable": 320,
"ndcg_at_10": 0.3983431061821887,
"recall_at_1": 0.21875,
"recall_at_10": 0.621875,
"recall_at_5": 0.465625,
"wrong_holder_confidence_unanswerable": 1.0
},
"video": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.7162790697674418,
"hit_at_10_wilson_lower": 0.6526351336615988,
"mrr": 0.3956736803248431,
"n_answerable": 215,
"n_unanswerable": 215,
"ndcg_at_10": 0.47027044859447076,
"recall_at_1": 0.2837209302325581,
"recall_at_10": 0.7162790697674418,
"recall_at_5": 0.5255813953488372,
"wrong_holder_confidence_unanswerable": 1.0
}
},
"closed_items": 0,
"items": 403859,
"latency_seconds": {
"max": 15.670968882004672,
"mean": 0.09444417075096513,
"p50": 0.08692938649983262,
"p95": 0.10732038790265505,
"queries": 5308
},
"overall": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.8674863387978142,
"hit_at_10_wilson_lower": 0.854722713774439,
"mrr": 0.6285142358400555,
"n_answerable": 2928,
"n_unanswerable": 2380,
"ndcg_at_10": 0.6852635110385413,
"recall_at_1": 0.49903802367941713,
"recall_at_10": 0.8664332877959927,
"recall_at_5": 0.7801286429872496,
"wrong_holder_confidence_unanswerable": 1.0
},
"store": "mem0_flat"
},
{
"by_modality": {
"code": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.952303961196443,
"hit_at_10_wilson_lower": 0.938963104367135,
"mrr": 0.6949699092787209,
"n_answerable": 1237,
"n_unanswerable": 1237,
"ndcg_at_10": 0.7581529489950144,
"recall_at_1": 0.549717057396928,
"recall_at_10": 0.952303961196443,
"recall_at_5": 0.8884397736459175,
"wrong_holder_confidence_unanswerable": 1.0
},
"conversation": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.11333333333333333,
"hit_at_10": 0.9866666666666667,
"hit_at_10_wilson_lower": 0.9526930912996325,
"mrr": 0.9053968253968253,
"n_answerable": 150,
"n_unanswerable": 150,
"ndcg_at_10": 0.9136591750866607,
"recall_at_1": 0.637888888888889,
"recall_at_10": 0.9783333333333334,
"recall_at_5": 0.9458888888888889,
"wrong_holder_confidence_unanswerable": 0.8866666666666667
},
"image": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.5,
"hit_at_10": 0.6541666666666667,
"hit_at_10_wilson_lower": 0.5919888820581466,
"mrr": 0.504917328042328,
"n_answerable": 240,
"n_unanswerable": 240,
"ndcg_at_10": 0.5355835725491713,
"recall_at_1": 0.42291666666666666,
"recall_at_10": 0.6458333333333334,
"recall_at_5": 0.6,
"wrong_holder_confidence_unanswerable": 0.5
},
"pdf": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.9164490861618799,
"hit_at_10_wilson_lower": 0.8947140126781276,
"mrr": 0.691579634464752,
"n_answerable": 766,
"n_unanswerable": 218,
"ndcg_at_10": 0.7454292373597072,
"recall_at_1": 0.587467362924282,
"recall_at_10": 0.9164490861618799,
"recall_at_5": 0.825065274151436,
"wrong_holder_confidence_unanswerable": 1.0
},
"ui_screenshot": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.621875,
"hit_at_10_wilson_lower": 0.567595266832662,
"mrr": 0.32982886904761904,
"n_answerable": 320,
"n_unanswerable": 320,
"ndcg_at_10": 0.3983431061821887,
"recall_at_1": 0.21875,
"recall_at_10": 0.621875,
"recall_at_5": 0.465625,
"wrong_holder_confidence_unanswerable": 1.0
},
"video": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.0,
"hit_at_10": 0.7162790697674418,
"hit_at_10_wilson_lower": 0.6526351336615988,
"mrr": 0.3956736803248431,
"n_answerable": 215,
"n_unanswerable": 215,
"ndcg_at_10": 0.47027044859447076,
"recall_at_1": 0.2837209302325581,
"recall_at_10": 0.7162790697674418,
"recall_at_5": 0.5255813953488372,
"wrong_holder_confidence_unanswerable": 1.0
}
},
"closed_items": 27072,
"items": 403859,
"latency_seconds": {
"max": 10.518811015004758,
"mean": 0.09140969146306362,
"p50": 0.0847548475030635,
"p95": 0.10844126994816179,
"queries": 5308
},
"overall": {
"abstention_rate_answerable": 0.0,
"abstention_rate_unanswerable": 0.05756302521008403,
"hit_at_10": 0.8668032786885246,
"hit_at_10_wilson_lower": 0.8540138593279664,
"mrr": 0.6274018236620695,
"n_answerable": 2928,
"n_unanswerable": 2380,
"ndcg_at_10": 0.6840849958419674,
"recall_at_1": 0.4980134335154827,
"recall_at_10": 0.865693306010929,
"recall_at_5": 0.7783071493624771,
"wrong_holder_confidence_unanswerable": 0.942436974789916
},
"store": "zep_temporal"
}
]
}
Reference in new issue View Git Blame Copy Permalink