ZacharyZhang-NY
a4953335e7
The merged database names a source by content hash (`11089_4f2a...`) while the frozen sets name the same file by its corpus id, so every arm scored 0.000 until load_atoms joined atoms through sources.path. It now returns corpus ids and refuses an atom it cannot trace to a corpus file. Also fixes found by running the harness: - merge rebuilt atom_fts from the deduped atoms table; two attempts on one file left 288,266 fts rows for 165,973 atoms. - merge accepts the two honest reasons a corpus file has no product source (500 byte-identical LongMemEval sessions collapse under content hashing, and 12 photos that ONNX Runtime cannot decode) and reports them. - --arms used action=extend against a non-empty default, so naming two arms ran seven. - the dense cache is keyed by the atom set, so a scale-curve subset cannot be scored with full-corpus vectors. - write_markdown crashed on a run that covered only some modalities.
77 lines
2.3 KiB
JSON
77 lines
2.3 KiB
JSON
{
|
|
"arms": [
|
|
{
|
|
"arm": "bm25_only",
|
|
"by_modality": {
|
|
"pdf": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 1.0,
|
|
"hit_at_10_wilson_lower": 0.6755924351161198,
|
|
"mrr": 0.725,
|
|
"n_answerable": 8,
|
|
"n_unanswerable": 2,
|
|
"ndcg_at_10": 0.7883559233904859,
|
|
"recall_at_1": 0.625,
|
|
"recall_at_10": 1.0,
|
|
"recall_at_5": 0.875,
|
|
"wrong_holder_confidence_unanswerable": 1.0
|
|
}
|
|
},
|
|
"latency_seconds": {
|
|
"max": 1.3251221780083142,
|
|
"mean": 0.8729710359009915,
|
|
"p50": 0.8812561564991483,
|
|
"p95": 1.2068847050097244,
|
|
"queries": 10
|
|
},
|
|
"overall": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 1.0,
|
|
"hit_at_10_wilson_lower": 0.6755924351161198,
|
|
"mrr": 0.725,
|
|
"n_answerable": 8,
|
|
"n_unanswerable": 2,
|
|
"ndcg_at_10": 0.7883559233904859,
|
|
"recall_at_1": 0.625,
|
|
"recall_at_10": 1.0,
|
|
"recall_at_5": 0.875,
|
|
"wrong_holder_confidence_unanswerable": 1.0
|
|
}
|
|
}
|
|
],
|
|
"chance": {
|
|
"corpus_sources": 50278,
|
|
"recall_at_1": 1.9889414853415013e-05,
|
|
"recall_at_10": 0.00019889414853415012,
|
|
"recall_at_5": 9.944707426707506e-05
|
|
},
|
|
"meta": {
|
|
"answerable": 8,
|
|
"corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347",
|
|
"corpus_sources": 50278,
|
|
"generated_at": "2026-10-06T20:42:42+00:00",
|
|
"git_commit": "fc0698c2426c373e32c361a816523351a1e3fcb7",
|
|
"limit_per_modality": 10,
|
|
"machine": {
|
|
"cpus": 32,
|
|
"release": "6.18.40.1-microsoft-standard-WSL2",
|
|
"system": "Linux"
|
|
},
|
|
"models": {
|
|
"dense": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
|
|
"rerank": "BAAI/bge-reranker-base"
|
|
},
|
|
"python": "3.14.4",
|
|
"queries": 10,
|
|
"sets": {
|
|
"code": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e",
|
|
"conversation": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11",
|
|
"image": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128",
|
|
"pdf": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536",
|
|
"ui_screenshot": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5",
|
|
"video": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c"
|
|
}
|
|
}
|
|
} |