Join the product's source ids to the benchmark's before scoring
The merged database names a source by content hash (`11089_4f2a...`) while the frozen sets name the same file by its corpus id, so every arm scored 0.000 until load_atoms joined atoms through sources.path. It now returns corpus ids and refuses an atom it cannot trace to a corpus file. Also fixes found by running the harness: - merge rebuilt atom_fts from the deduped atoms table; two attempts on one file left 288,266 fts rows for 165,973 atoms. - merge accepts the two honest reasons a corpus file has no product source (500 byte-identical LongMemEval sessions collapse under content hashing, and 12 photos that ONNX Runtime cannot decode) and reports them. - --arms used action=extend against a non-empty default, so naming two arms ran seven. - the dense cache is keyed by the atom set, so a scale-curve subset cannot be scored with full-corpus vectors. - write_markdown crashed on a run that covered only some modalities.
This commit is contained in:
1 parent
fc0698c242
commit
a4953335e7
7 files changed
+238
-26
No files matched your search
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"arms": [
|
||||
{
|
||||
"arm": "bm25_only",
|
||||
"by_modality": {
|
||||
"pdf": {
|
||||
"abstention_rate_answerable": 0.0,
|
||||
"abstention_rate_unanswerable": 0.0,
|
||||
"hit_at_10": 1.0,
|
||||
"hit_at_10_wilson_lower": 0.6755924351161198,
|
||||
"mrr": 0.725,
|
||||
"n_answerable": 8,
|
||||
"n_unanswerable": 2,
|
||||
"ndcg_at_10": 0.7883559233904859,
|
||||
"recall_at_1": 0.625,
|
||||
"recall_at_10": 1.0,
|
||||
"recall_at_5": 0.875,
|
||||
"wrong_holder_confidence_unanswerable": 1.0
|
||||
}
|
||||
},
|
||||
"latency_seconds": {
|
||||
"max": 1.3251221780083142,
|
||||
"mean": 0.8729710359009915,
|
||||
"p50": 0.8812561564991483,
|
||||
"p95": 1.2068847050097244,
|
||||
"queries": 10
|
||||
},
|
||||
"overall": {
|
||||
"abstention_rate_answerable": 0.0,
|
||||
"abstention_rate_unanswerable": 0.0,
|
||||
"hit_at_10": 1.0,
|
||||
"hit_at_10_wilson_lower": 0.6755924351161198,
|
||||
"mrr": 0.725,
|
||||
"n_answerable": 8,
|
||||
"n_unanswerable": 2,
|
||||
"ndcg_at_10": 0.7883559233904859,
|
||||
"recall_at_1": 0.625,
|
||||
"recall_at_10": 1.0,
|
||||
"recall_at_5": 0.875,
|
||||
"wrong_holder_confidence_unanswerable": 1.0
|
||||
}
|
||||
}
|
||||
],
|
||||
"chance": {
|
||||
"corpus_sources": 50278,
|
||||
"recall_at_1": 1.9889414853415013e-05,
|
||||
"recall_at_10": 0.00019889414853415012,
|
||||
"recall_at_5": 9.944707426707506e-05
|
||||
},
|
||||
"meta": {
|
||||
"answerable": 8,
|
||||
"corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347",
|
||||
"corpus_sources": 50278,
|
||||
"generated_at": "2026-10-06T20:42:42+00:00",
|
||||
"git_commit": "fc0698c2426c373e32c361a816523351a1e3fcb7",
|
||||
"limit_per_modality": 10,
|
||||
"machine": {
|
||||
"cpus": 32,
|
||||
"release": "6.18.40.1-microsoft-standard-WSL2",
|
||||
"system": "Linux"
|
||||
},
|
||||
"models": {
|
||||
"dense": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
|
||||
"rerank": "BAAI/bge-reranker-base"
|
||||
},
|
||||
"python": "3.14.4",
|
||||
"queries": 10,
|
||||
"sets": {
|
||||
"code": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e",
|
||||
"conversation": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11",
|
||||
"image": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128",
|
||||
"pdf": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536",
|
||||
"ui_screenshot": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5",
|
||||
"video": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user