{ "arms": [ { "arm": "bm25_only", "by_modality": { "code": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.5181891673403395, "hit_at_10_wilson_lower": 0.4903310330926623, "mrr": 0.36655150068650477, "n_answerable": 1237, "n_unanswerable": 1237, "ndcg_at_10": 0.4030722360859963, "recall_at_1": 0.29345189975747776, "recall_at_10": 0.5181891673403395, "recall_at_5": 0.46160064672594986, "top_hit_from_holder_unanswerable": 0.629749393694422 }, "conversation": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.4533333333333333, "hit_at_10_wilson_lower": 0.37582498607927695, "mrr": 0.2632063492063492, "n_answerable": 150, "n_unanswerable": 150, "ndcg_at_10": 0.2739998550294506, "recall_at_1": 0.16333333333333333, "recall_at_10": 0.36822222222222223, "recall_at_5": 0.31222222222222223, "top_hit_from_holder_unanswerable": 0.006666666666666667 }, "image": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.6458333333333334, "hit_at_10_wilson_lower": 0.5834633143794732, "mrr": 0.5017857142857143, "n_answerable": 240, "n_unanswerable": 240, "ndcg_at_10": 0.5293922693217483, "recall_at_1": 0.42083333333333334, "recall_at_10": 0.6375, "recall_at_5": 0.575, "top_hit_from_holder_unanswerable": 0.0 }, "pdf": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.7036553524804178, "hit_at_10_wilson_lower": 0.6703659725843993, "mrr": 0.564516246011024, "n_answerable": 766, "n_unanswerable": 218, "ndcg_at_10": 0.5982115197161956, "recall_at_1": 0.4934725848563969, "recall_at_10": 0.7036553524804178, "recall_at_5": 0.6540469973890339, "top_hit_from_holder_unanswerable": 0.5688073394495413 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.321875, "hit_at_10_wilson_lower": 0.27306019860147723, "mrr": 0.1832576884920635, "n_answerable": 320, "n_unanswerable": 320, "ndcg_at_10": 0.21641192936033926, "recall_at_1": 0.11875, "recall_at_10": 0.321875, "recall_at_5": 0.2625, "top_hit_from_holder_unanswerable": 0.015625 }, "video": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.12558139534883722, "hit_at_10_wilson_lower": 0.08776042446405018, "mrr": 0.08080103359173127, "n_answerable": 215, "n_unanswerable": 215, "ndcg_at_10": 0.09134653357120204, "recall_at_1": 0.06046511627906977, "recall_at_10": 0.12558139534883722, "recall_at_5": 0.10232558139534884, "top_hit_from_holder_unanswerable": 0.004651162790697674 } }, "latency_seconds": { "max": 70.10108924399901, "mean": 1.033756499582343, "p50": 0.5625866459999997, "p95": 3.1407018873005645, "queries": 5308 }, "overall": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.5235655737704918, "hit_at_10_wilson_lower": 0.5054560637114823, "mrr": 0.38311743754879, "n_answerable": 2928, "n_unanswerable": 2380, "ndcg_at_10": 0.41457507689403594, "recall_at_1": 0.31335382513661203, "recall_at_10": 0.5185223132969035, "recall_at_5": 0.4654485428051002, "top_hit_from_holder_unanswerable": 0.38235294117647056 } }, { "arm": "dense_only", "by_modality": { "code": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.38156831042845596, "hit_at_10_wilson_lower": 0.3549039694490394, "mrr": 0.23736478423220542, "n_answerable": 1237, "n_unanswerable": 1237, "ndcg_at_10": 0.2717431059401204, "recall_at_1": 0.17623282134195634, "recall_at_10": 0.38156831042845596, "recall_at_5": 0.3193209377526273, "top_hit_from_holder_unanswerable": 0.37510105092966856 }, "conversation": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.38666666666666666, "hit_at_10_wilson_lower": 0.31249116779192654, "mrr": 0.2273148148148148, "n_answerable": 150, "n_unanswerable": 150, "ndcg_at_10": 0.2312801291223675, "recall_at_1": 0.12666666666666668, "recall_at_10": 0.3042222222222222, "recall_at_5": 0.26466666666666666, "top_hit_from_holder_unanswerable": 0.0 }, "image": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.4875, "hit_at_10_wilson_lower": 0.4249589194528996, "mrr": 0.3143535052910053, "n_answerable": 240, "n_unanswerable": 240, "ndcg_at_10": 0.35184243019735995, "recall_at_1": 0.21875, "recall_at_10": 0.48125, "recall_at_5": 0.42291666666666666, "top_hit_from_holder_unanswerable": 0.0125 }, "pdf": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.6984334203655352, "hit_at_10_wilson_lower": 0.6650089879505938, "mrr": 0.5372964068133781, "n_answerable": 766, "n_unanswerable": 218, "ndcg_at_10": 0.5759767947831819, "recall_at_1": 0.4608355091383812, "recall_at_10": 0.6984334203655352, "recall_at_5": 0.6292428198433421, "top_hit_from_holder_unanswerable": 0.5779816513761468 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.075, "hit_at_10_wilson_lower": 0.05091491480619657, "mrr": 0.030979662698412697, "n_answerable": 320, "n_unanswerable": 320, "ndcg_at_10": 0.04122199724982832, "recall_at_1": 0.015625, "recall_at_10": 0.075, "recall_at_5": 0.046875, "top_hit_from_holder_unanswerable": 0.0 }, "video": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.22790697674418606, "hit_at_10_wilson_lower": 0.1769011082994195, "mrr": 0.1167921742340347, "n_answerable": 215, "n_unanswerable": 215, "ndcg_at_10": 0.14219297062759328, "recall_at_1": 0.08372093023255814, "recall_at_10": 0.22790697674418606, "recall_at_5": 0.14418604651162792, "top_hit_from_holder_unanswerable": 0.013953488372093023 } }, "latency_seconds": { "max": 5.211815150996699, "mean": 1.90850240464544, "p50": 1.7281245969993506, "p95": 2.8826986088994087, "queries": 5308 }, "overall": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.428620218579235, "hit_at_10_wilson_lower": 0.41080015838413203, "mrr": 0.29021692579581926, "n_answerable": 2928, "n_unanswerable": 2380, "ndcg_at_10": 0.32112062065315755, "recall_at_1": 0.22728825136612021, "recall_at_10": 0.423884335154827, "recall_at_5": 0.36345628415300546, "top_hit_from_holder_unanswerable": 0.2504201680672269 } }, { "arm": "graph_only", "by_modality": { "code": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0016168148746968471, "hit_at_10_wilson_lower": 0.0004435009881340089, "mrr": 0.0009700889248181083, "n_answerable": 1237, "n_unanswerable": 1237, "ndcg_at_10": 0.001121142123875943, "recall_at_1": 0.0008084074373484236, "recall_at_10": 0.0016168148746968471, "recall_at_5": 0.0016168148746968471, "top_hit_from_holder_unanswerable": 0.0 }, "conversation": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0, "hit_at_10_wilson_lower": 0.0, "mrr": 0.0, "n_answerable": 150, "n_unanswerable": 150, "ndcg_at_10": 0.0, "recall_at_1": 0.0, "recall_at_10": 0.0, "recall_at_5": 0.0, "top_hit_from_holder_unanswerable": 0.0 }, "image": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0, "hit_at_10_wilson_lower": 0.0, "mrr": 0.0, "n_answerable": 240, "n_unanswerable": 240, "ndcg_at_10": 0.0, "recall_at_1": 0.0, "recall_at_10": 0.0, "recall_at_5": 0.0, "top_hit_from_holder_unanswerable": 0.0 }, "pdf": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0, "hit_at_10_wilson_lower": 0.0, "mrr": 0.0, "n_answerable": 766, "n_unanswerable": 218, "ndcg_at_10": 0.0, "recall_at_1": 0.0, "recall_at_10": 0.0, "recall_at_5": 0.0, "top_hit_from_holder_unanswerable": 0.0 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0, "hit_at_10_wilson_lower": 0.0, "mrr": 0.0, "n_answerable": 320, "n_unanswerable": 320, "ndcg_at_10": 0.0, "recall_at_1": 0.0, "recall_at_10": 0.0, "recall_at_5": 0.0, "top_hit_from_holder_unanswerable": 0.0 }, "video": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0, "hit_at_10_wilson_lower": 0.0, "mrr": 0.0, "n_answerable": 215, "n_unanswerable": 215, "ndcg_at_10": 0.0, "recall_at_1": 0.0, "recall_at_10": 0.0, "recall_at_5": 0.0, "top_hit_from_holder_unanswerable": 0.0 } }, "latency_seconds": { "max": 0.09835406600177521, "mean": 0.044743228177486605, "p50": 0.04287036849927972, "p95": 0.05790443480054818, "queries": 5308 }, "overall": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.0006830601092896175, "hit_at_10_wilson_lower": 0.00018733970917281023, "mrr": 0.0004098360655737705, "n_answerable": 2928, "n_unanswerable": 2380, "ndcg_at_10": 0.0004736519150391194, "recall_at_1": 0.00034153005464480874, "recall_at_10": 0.0006830601092896175, "recall_at_5": 0.0006830601092896175, "top_hit_from_holder_unanswerable": 0.0 } }, { "arm": "no_rerank", "by_modality": { "code": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.5917542441390461, "hit_at_10_wilson_lower": 0.5641209696430517, "mrr": 0.4132014217705406, "n_answerable": 1237, "n_unanswerable": 1237, "ndcg_at_10": 0.4559664301805668, "recall_at_1": 0.3314470493128537, "recall_at_10": 0.5917542441390461, "recall_at_5": 0.52303961196443, "top_hit_from_holder_unanswerable": 0.666127728375101 }, "conversation": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.54, "hit_at_10_wilson_lower": 0.46023821464666953, "mrr": 0.3229312169312169, "n_answerable": 150, "n_unanswerable": 150, "ndcg_at_10": 0.3349165804130411, "recall_at_1": 0.19, "recall_at_10": 0.44711111111111107, "recall_at_5": 0.38155555555555554, "top_hit_from_holder_unanswerable": 0.0 }, "image": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.675, "hit_at_10_wilson_lower": 0.6133905629314187, "mrr": 0.5472437169312169, "n_answerable": 240, "n_unanswerable": 240, "ndcg_at_10": 0.5716072495855514, "recall_at_1": 0.4791666666666667, "recall_at_10": 0.66875, "recall_at_5": 0.6, "top_hit_from_holder_unanswerable": 0.0 }, "pdf": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.741514360313316, "hit_at_10_wilson_lower": 0.7093596270841578, "mrr": 0.5952085664553027, "n_answerable": 766, "n_unanswerable": 218, "ndcg_at_10": 0.6306490300771556, "recall_at_1": 0.5195822454308094, "recall_at_10": 0.741514360313316, "recall_at_5": 0.695822454308094, "top_hit_from_holder_unanswerable": 0.6146788990825688 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.296875, "hit_at_10_wilson_lower": 0.24946567415196566, "mrr": 0.1761359126984127, "n_answerable": 320, "n_unanswerable": 320, "ndcg_at_10": 0.20522256852542595, "recall_at_1": 0.11875, "recall_at_10": 0.296875, "recall_at_5": 0.259375, "top_hit_from_holder_unanswerable": 0.009375 }, "video": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.15348837209302327, "hit_at_10_wilson_lower": 0.11142806397476775, "mrr": 0.0915891472868217, "n_answerable": 215, "n_unanswerable": 215, "ndcg_at_10": 0.10659041518078256, "recall_at_1": 0.06046511627906977, "recall_at_10": 0.15348837209302327, "recall_at_5": 0.13953488372093023, "top_hit_from_holder_unanswerable": 0.004651162790697674 } }, "latency_seconds": { "max": 72.48293514099714, "mean": 2.882459499187649, "p50": 2.40399285599824, "p95": 5.168583710499833, "queries": 5308 }, "overall": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.5706967213114754, "hit_at_10_wilson_lower": 0.552686960572952, "mrr": 0.4176547998525457, "n_answerable": 2928, "n_unanswerable": 2380, "ndcg_at_10": 0.4518849109723208, "recall_at_1": 0.34238387978142076, "recall_at_10": 0.5654257741347906, "recall_at_5": 0.510325591985428, "top_hit_from_holder_unanswerable": 0.4042016806722689 } }, { "arm": "fusion_rerank", "by_modality": { "code": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.6289409862570736, "hit_at_10_wilson_lower": 0.6016596375208901, "mrr": 0.4450988695127741, "n_answerable": 1237, "n_unanswerable": 1237, "ndcg_at_10": 0.48964513720914205, "recall_at_1": 0.35084882780921584, "recall_at_10": 0.6289409862570736, "recall_at_5": 0.5675020210185934, "top_hit_from_holder_unanswerable": 0.6313662085691188 }, "conversation": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.5866666666666667, "hit_at_10_wilson_lower": 0.5066585675662262, "mrr": 0.44420105820105815, "n_answerable": 150, "n_unanswerable": 150, "ndcg_at_10": 0.41993124265105486, "recall_at_1": 0.28, "recall_at_10": 0.4862222222222222, "recall_at_5": 0.42911111111111105, "top_hit_from_holder_unanswerable": 0.006666666666666667 }, "image": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.7, "hit_at_10_wilson_lower": 0.6392449572501318, "mrr": 0.5878356481481481, "n_answerable": 240, "n_unanswerable": 240, "ndcg_at_10": 0.6098813565843775, "recall_at_1": 0.5208333333333334, "recall_at_10": 0.6916666666666667, "recall_at_5": 0.6520833333333333, "top_hit_from_holder_unanswerable": 0.016666666666666666 }, "pdf": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.25326370757180156, "hit_at_10_wilson_lower": 0.223750502086736, "mrr": 0.1029870695014298, "n_answerable": 766, "n_unanswerable": 218, "ndcg_at_10": 0.13796388574710278, "recall_at_1": 0.05221932114882506, "recall_at_10": 0.25326370757180156, "recall_at_5": 0.16579634464751958, "top_hit_from_holder_unanswerable": 0.0871559633027523 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.3, "hit_at_10_wilson_lower": 0.25240562491350604, "mrr": 0.18429935515873014, "n_answerable": 320, "n_unanswerable": 320, "ndcg_at_10": 0.21181060150559686, "recall_at_1": 0.1375, "recall_at_10": 0.3, "recall_at_5": 0.25625, "top_hit_from_holder_unanswerable": 0.03125 }, "video": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.08837209302325581, "hit_at_10_wilson_lower": 0.05730442391262212, "mrr": 0.04147840531561462, "n_answerable": 215, "n_unanswerable": 215, "ndcg_at_10": 0.0524917978865275, "recall_at_1": 0.023255813953488372, "recall_at_10": 0.08837209302325581, "recall_at_5": 0.06511627906976744, "top_hit_from_holder_unanswerable": 0.0 } }, "latency_seconds": { "max": 77.63319287999548, "mean": 5.166508504176891, "p50": 4.5375522020012795, "p95": 9.042528758149817, "queries": 5308 }, "overall": { "abstention_rate_answerable": 0.0, "abstention_rate_unanswerable": 0.0, "hit_at_10": 0.45867486338797814, "hit_at_10_wilson_lower": 0.4406921315699621, "mrr": 0.3091119405412438, "n_answerable": 2928, "n_unanswerable": 2380, "ndcg_at_10": 0.34146096728664366, "recall_at_1": 0.23565573770491804, "recall_at_10": 0.4528460837887068, "recall_at_5": 0.3913479052823315, "top_hit_from_holder_unanswerable": 0.34243697478991597 } } ], "chance": { "corpus_sources": 50278, "recall_at_1": 1.9889414853415013e-05, "recall_at_10": 0.00019889414853415012, "recall_at_5": 9.944707426707506e-05 }, "live_product": { "abstention_rate_answerable": 0.016666666666666666, "abstention_rate_unanswerable": 0.0, "distinct_sources_in_top10": { "max": 10, "mean": 5.193333333333333, "min": 0 }, "hit_at_10": 0.8333333333333334, "hit_at_10_wilson_lower": 0.7870033503873167, "identical_top10": 0.0, "max": 0.3143954080005642, "mean": 0.07851241317994816, "mrr": 0.6832447089947089, "n_answerable": 300, "n_unanswerable": 0, "ndcg_at_10": 0.7186678703708246, "p50": 0.07425041199894622, "p95": 0.14573381999798585, "queries": 300, "recall_at_1": 0.5966666666666667, "recall_at_10": 0.8316666666666667, "recall_at_5": 0.7816666666666666, "top10_overlap_with_bm25_only": 0.13433333333333336, "top_hit_from_holder_unanswerable": 0.0, "top_k": 10 }, "meta": { "answerable": 2928, "corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347", "corpus_sources": 50278, "generated_at": "2026-10-08T20:17:12+00:00", "git_commit": "dff5977ab638896cc8685c78ee41e5195fc0fbf2", "limit_per_modality": null, "machine": { "cpus": 32, "load_average_1_5_15": [ 5.09, 4.28, 8.06 ], "release": "6.18.40.1-microsoft-standard-WSL2", "system": "Linux", "threads_per_query": 4 }, "models": { "dense": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "rerank": "BAAI/bge-reranker-base" }, "python": "3.14.4", "queries": 5308, "sets": { "code": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e", "conversation": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11", "image": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128", "pdf": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536", "ui_screenshot": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5", "video": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c" }, "store_schema_version_at_start": 8 } }