ZacharyZhang-NY
c4a8c36416
Every number in CHECKLIST p2-1 and p0-3.7 came out of a scratch script: the
kind of artifact a reader cannot re-run, and the kind that quietly drifts
away from the code it describes. These reports are generated by committed
harnesses and each one stamps the commit that carries the code it measures.
DENSE.md 165,973 atoms / 153,857 vectors: build 1421.0s, search p50 75.6ms
p95 93.0ms, 0.998 agreement with a brute-force scan, one write into
a full index p50 518ms, 1,729 bytes per stored vector.
QUANT.md twelve encodings measured on the same vectors: int8 keeps 0.977 of
the ranking for a quarter of the bytes and no speed, bit is 14x
faster and keeps 0.728, so bit is a coarse filter and nothing more.
TERMS.md the stop-word bound swept, not guessed: 0.05 is the measured peak
(recall@10 0.803 at p50 69.5ms) and also the fastest of the rows
that keep recall.
SCALE.md the product's own search at 165,973 atoms: p50 71.7ms against the
frozen baseline's 20.16s.
LEGS.md what each retrieval leg is worth, per modality, with the page size
the product actually asks for in the table.
The acceptance bar the operator set is recall@10 >= 0.90 at n>=300. It is
not met. The best measured row is 0.858 (dense leg on, page widened to 30,
p50 414ms); the shipped configuration gives 0.848 at p50 72.5ms; the
in-repo counterpart at holder scope gives 0.866. The shortfall is
concentrated in picture golds (video 0.583-0.708, ui_screenshot 0.500-0.583,
image 0.750) while the word-shaped ones already pass (code 0.953, pdf 0.919).
Latency passes by three hundred fold; recall does not, and the gap is four
points of cross-modal retrieval, not index scale.
621 lines
19 KiB
JSON
621 lines
19 KiB
JSON
{
|
|
"arms": [
|
|
{
|
|
"arm": "bm25_only",
|
|
"by_modality": {
|
|
"code": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.5181891673403395,
|
|
"hit_at_10_wilson_lower": 0.4903310330926623,
|
|
"mrr": 0.36655150068650477,
|
|
"n_answerable": 1237,
|
|
"n_unanswerable": 1237,
|
|
"ndcg_at_10": 0.4030722360859963,
|
|
"recall_at_1": 0.29345189975747776,
|
|
"recall_at_10": 0.5181891673403395,
|
|
"recall_at_5": 0.46160064672594986,
|
|
"top_hit_from_holder_unanswerable": 0.629749393694422
|
|
},
|
|
"conversation": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.4533333333333333,
|
|
"hit_at_10_wilson_lower": 0.37582498607927695,
|
|
"mrr": 0.2632063492063492,
|
|
"n_answerable": 150,
|
|
"n_unanswerable": 150,
|
|
"ndcg_at_10": 0.2739998550294506,
|
|
"recall_at_1": 0.16333333333333333,
|
|
"recall_at_10": 0.36822222222222223,
|
|
"recall_at_5": 0.31222222222222223,
|
|
"top_hit_from_holder_unanswerable": 0.006666666666666667
|
|
},
|
|
"image": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.6458333333333334,
|
|
"hit_at_10_wilson_lower": 0.5834633143794732,
|
|
"mrr": 0.5017857142857143,
|
|
"n_answerable": 240,
|
|
"n_unanswerable": 240,
|
|
"ndcg_at_10": 0.5293922693217483,
|
|
"recall_at_1": 0.42083333333333334,
|
|
"recall_at_10": 0.6375,
|
|
"recall_at_5": 0.575,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"pdf": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.7036553524804178,
|
|
"hit_at_10_wilson_lower": 0.6703659725843993,
|
|
"mrr": 0.564516246011024,
|
|
"n_answerable": 766,
|
|
"n_unanswerable": 218,
|
|
"ndcg_at_10": 0.5982115197161956,
|
|
"recall_at_1": 0.4934725848563969,
|
|
"recall_at_10": 0.7036553524804178,
|
|
"recall_at_5": 0.6540469973890339,
|
|
"top_hit_from_holder_unanswerable": 0.5688073394495413
|
|
},
|
|
"ui_screenshot": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.321875,
|
|
"hit_at_10_wilson_lower": 0.27306019860147723,
|
|
"mrr": 0.1832576884920635,
|
|
"n_answerable": 320,
|
|
"n_unanswerable": 320,
|
|
"ndcg_at_10": 0.21641192936033926,
|
|
"recall_at_1": 0.11875,
|
|
"recall_at_10": 0.321875,
|
|
"recall_at_5": 0.2625,
|
|
"top_hit_from_holder_unanswerable": 0.015625
|
|
},
|
|
"video": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.12558139534883722,
|
|
"hit_at_10_wilson_lower": 0.08776042446405018,
|
|
"mrr": 0.08080103359173127,
|
|
"n_answerable": 215,
|
|
"n_unanswerable": 215,
|
|
"ndcg_at_10": 0.09134653357120204,
|
|
"recall_at_1": 0.06046511627906977,
|
|
"recall_at_10": 0.12558139534883722,
|
|
"recall_at_5": 0.10232558139534884,
|
|
"top_hit_from_holder_unanswerable": 0.004651162790697674
|
|
}
|
|
},
|
|
"latency_seconds": {
|
|
"max": 70.10108924399901,
|
|
"mean": 1.033756499582343,
|
|
"p50": 0.5625866459999997,
|
|
"p95": 3.1407018873005645,
|
|
"queries": 5308
|
|
},
|
|
"overall": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.5235655737704918,
|
|
"hit_at_10_wilson_lower": 0.5054560637114823,
|
|
"mrr": 0.38311743754879,
|
|
"n_answerable": 2928,
|
|
"n_unanswerable": 2380,
|
|
"ndcg_at_10": 0.41457507689403594,
|
|
"recall_at_1": 0.31335382513661203,
|
|
"recall_at_10": 0.5185223132969035,
|
|
"recall_at_5": 0.4654485428051002,
|
|
"top_hit_from_holder_unanswerable": 0.38235294117647056
|
|
}
|
|
},
|
|
{
|
|
"arm": "dense_only",
|
|
"by_modality": {
|
|
"code": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.38156831042845596,
|
|
"hit_at_10_wilson_lower": 0.3549039694490394,
|
|
"mrr": 0.23736478423220542,
|
|
"n_answerable": 1237,
|
|
"n_unanswerable": 1237,
|
|
"ndcg_at_10": 0.2717431059401204,
|
|
"recall_at_1": 0.17623282134195634,
|
|
"recall_at_10": 0.38156831042845596,
|
|
"recall_at_5": 0.3193209377526273,
|
|
"top_hit_from_holder_unanswerable": 0.37510105092966856
|
|
},
|
|
"conversation": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.38666666666666666,
|
|
"hit_at_10_wilson_lower": 0.31249116779192654,
|
|
"mrr": 0.2273148148148148,
|
|
"n_answerable": 150,
|
|
"n_unanswerable": 150,
|
|
"ndcg_at_10": 0.2312801291223675,
|
|
"recall_at_1": 0.12666666666666668,
|
|
"recall_at_10": 0.3042222222222222,
|
|
"recall_at_5": 0.26466666666666666,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"image": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.4875,
|
|
"hit_at_10_wilson_lower": 0.4249589194528996,
|
|
"mrr": 0.3143535052910053,
|
|
"n_answerable": 240,
|
|
"n_unanswerable": 240,
|
|
"ndcg_at_10": 0.35184243019735995,
|
|
"recall_at_1": 0.21875,
|
|
"recall_at_10": 0.48125,
|
|
"recall_at_5": 0.42291666666666666,
|
|
"top_hit_from_holder_unanswerable": 0.0125
|
|
},
|
|
"pdf": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.6984334203655352,
|
|
"hit_at_10_wilson_lower": 0.6650089879505938,
|
|
"mrr": 0.5372964068133781,
|
|
"n_answerable": 766,
|
|
"n_unanswerable": 218,
|
|
"ndcg_at_10": 0.5759767947831819,
|
|
"recall_at_1": 0.4608355091383812,
|
|
"recall_at_10": 0.6984334203655352,
|
|
"recall_at_5": 0.6292428198433421,
|
|
"top_hit_from_holder_unanswerable": 0.5779816513761468
|
|
},
|
|
"ui_screenshot": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.075,
|
|
"hit_at_10_wilson_lower": 0.05091491480619657,
|
|
"mrr": 0.030979662698412697,
|
|
"n_answerable": 320,
|
|
"n_unanswerable": 320,
|
|
"ndcg_at_10": 0.04122199724982832,
|
|
"recall_at_1": 0.015625,
|
|
"recall_at_10": 0.075,
|
|
"recall_at_5": 0.046875,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"video": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.22790697674418606,
|
|
"hit_at_10_wilson_lower": 0.1769011082994195,
|
|
"mrr": 0.1167921742340347,
|
|
"n_answerable": 215,
|
|
"n_unanswerable": 215,
|
|
"ndcg_at_10": 0.14219297062759328,
|
|
"recall_at_1": 0.08372093023255814,
|
|
"recall_at_10": 0.22790697674418606,
|
|
"recall_at_5": 0.14418604651162792,
|
|
"top_hit_from_holder_unanswerable": 0.013953488372093023
|
|
}
|
|
},
|
|
"latency_seconds": {
|
|
"max": 5.211815150996699,
|
|
"mean": 1.90850240464544,
|
|
"p50": 1.7281245969993506,
|
|
"p95": 2.8826986088994087,
|
|
"queries": 5308
|
|
},
|
|
"overall": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.428620218579235,
|
|
"hit_at_10_wilson_lower": 0.41080015838413203,
|
|
"mrr": 0.29021692579581926,
|
|
"n_answerable": 2928,
|
|
"n_unanswerable": 2380,
|
|
"ndcg_at_10": 0.32112062065315755,
|
|
"recall_at_1": 0.22728825136612021,
|
|
"recall_at_10": 0.423884335154827,
|
|
"recall_at_5": 0.36345628415300546,
|
|
"top_hit_from_holder_unanswerable": 0.2504201680672269
|
|
}
|
|
},
|
|
{
|
|
"arm": "graph_only",
|
|
"by_modality": {
|
|
"code": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0016168148746968471,
|
|
"hit_at_10_wilson_lower": 0.0004435009881340089,
|
|
"mrr": 0.0009700889248181083,
|
|
"n_answerable": 1237,
|
|
"n_unanswerable": 1237,
|
|
"ndcg_at_10": 0.001121142123875943,
|
|
"recall_at_1": 0.0008084074373484236,
|
|
"recall_at_10": 0.0016168148746968471,
|
|
"recall_at_5": 0.0016168148746968471,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"conversation": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0,
|
|
"hit_at_10_wilson_lower": 0.0,
|
|
"mrr": 0.0,
|
|
"n_answerable": 150,
|
|
"n_unanswerable": 150,
|
|
"ndcg_at_10": 0.0,
|
|
"recall_at_1": 0.0,
|
|
"recall_at_10": 0.0,
|
|
"recall_at_5": 0.0,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"image": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0,
|
|
"hit_at_10_wilson_lower": 0.0,
|
|
"mrr": 0.0,
|
|
"n_answerable": 240,
|
|
"n_unanswerable": 240,
|
|
"ndcg_at_10": 0.0,
|
|
"recall_at_1": 0.0,
|
|
"recall_at_10": 0.0,
|
|
"recall_at_5": 0.0,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"pdf": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0,
|
|
"hit_at_10_wilson_lower": 0.0,
|
|
"mrr": 0.0,
|
|
"n_answerable": 766,
|
|
"n_unanswerable": 218,
|
|
"ndcg_at_10": 0.0,
|
|
"recall_at_1": 0.0,
|
|
"recall_at_10": 0.0,
|
|
"recall_at_5": 0.0,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"ui_screenshot": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0,
|
|
"hit_at_10_wilson_lower": 0.0,
|
|
"mrr": 0.0,
|
|
"n_answerable": 320,
|
|
"n_unanswerable": 320,
|
|
"ndcg_at_10": 0.0,
|
|
"recall_at_1": 0.0,
|
|
"recall_at_10": 0.0,
|
|
"recall_at_5": 0.0,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"video": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0,
|
|
"hit_at_10_wilson_lower": 0.0,
|
|
"mrr": 0.0,
|
|
"n_answerable": 215,
|
|
"n_unanswerable": 215,
|
|
"ndcg_at_10": 0.0,
|
|
"recall_at_1": 0.0,
|
|
"recall_at_10": 0.0,
|
|
"recall_at_5": 0.0,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
}
|
|
},
|
|
"latency_seconds": {
|
|
"max": 0.09835406600177521,
|
|
"mean": 0.044743228177486605,
|
|
"p50": 0.04287036849927972,
|
|
"p95": 0.05790443480054818,
|
|
"queries": 5308
|
|
},
|
|
"overall": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.0006830601092896175,
|
|
"hit_at_10_wilson_lower": 0.00018733970917281023,
|
|
"mrr": 0.0004098360655737705,
|
|
"n_answerable": 2928,
|
|
"n_unanswerable": 2380,
|
|
"ndcg_at_10": 0.0004736519150391194,
|
|
"recall_at_1": 0.00034153005464480874,
|
|
"recall_at_10": 0.0006830601092896175,
|
|
"recall_at_5": 0.0006830601092896175,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
}
|
|
},
|
|
{
|
|
"arm": "no_rerank",
|
|
"by_modality": {
|
|
"code": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.5917542441390461,
|
|
"hit_at_10_wilson_lower": 0.5641209696430517,
|
|
"mrr": 0.4132014217705406,
|
|
"n_answerable": 1237,
|
|
"n_unanswerable": 1237,
|
|
"ndcg_at_10": 0.4559664301805668,
|
|
"recall_at_1": 0.3314470493128537,
|
|
"recall_at_10": 0.5917542441390461,
|
|
"recall_at_5": 0.52303961196443,
|
|
"top_hit_from_holder_unanswerable": 0.666127728375101
|
|
},
|
|
"conversation": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.54,
|
|
"hit_at_10_wilson_lower": 0.46023821464666953,
|
|
"mrr": 0.3229312169312169,
|
|
"n_answerable": 150,
|
|
"n_unanswerable": 150,
|
|
"ndcg_at_10": 0.3349165804130411,
|
|
"recall_at_1": 0.19,
|
|
"recall_at_10": 0.44711111111111107,
|
|
"recall_at_5": 0.38155555555555554,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"image": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.675,
|
|
"hit_at_10_wilson_lower": 0.6133905629314187,
|
|
"mrr": 0.5472437169312169,
|
|
"n_answerable": 240,
|
|
"n_unanswerable": 240,
|
|
"ndcg_at_10": 0.5716072495855514,
|
|
"recall_at_1": 0.4791666666666667,
|
|
"recall_at_10": 0.66875,
|
|
"recall_at_5": 0.6,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
},
|
|
"pdf": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.741514360313316,
|
|
"hit_at_10_wilson_lower": 0.7093596270841578,
|
|
"mrr": 0.5952085664553027,
|
|
"n_answerable": 766,
|
|
"n_unanswerable": 218,
|
|
"ndcg_at_10": 0.6306490300771556,
|
|
"recall_at_1": 0.5195822454308094,
|
|
"recall_at_10": 0.741514360313316,
|
|
"recall_at_5": 0.695822454308094,
|
|
"top_hit_from_holder_unanswerable": 0.6146788990825688
|
|
},
|
|
"ui_screenshot": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.296875,
|
|
"hit_at_10_wilson_lower": 0.24946567415196566,
|
|
"mrr": 0.1761359126984127,
|
|
"n_answerable": 320,
|
|
"n_unanswerable": 320,
|
|
"ndcg_at_10": 0.20522256852542595,
|
|
"recall_at_1": 0.11875,
|
|
"recall_at_10": 0.296875,
|
|
"recall_at_5": 0.259375,
|
|
"top_hit_from_holder_unanswerable": 0.009375
|
|
},
|
|
"video": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.15348837209302327,
|
|
"hit_at_10_wilson_lower": 0.11142806397476775,
|
|
"mrr": 0.0915891472868217,
|
|
"n_answerable": 215,
|
|
"n_unanswerable": 215,
|
|
"ndcg_at_10": 0.10659041518078256,
|
|
"recall_at_1": 0.06046511627906977,
|
|
"recall_at_10": 0.15348837209302327,
|
|
"recall_at_5": 0.13953488372093023,
|
|
"top_hit_from_holder_unanswerable": 0.004651162790697674
|
|
}
|
|
},
|
|
"latency_seconds": {
|
|
"max": 72.48293514099714,
|
|
"mean": 2.882459499187649,
|
|
"p50": 2.40399285599824,
|
|
"p95": 5.168583710499833,
|
|
"queries": 5308
|
|
},
|
|
"overall": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.5706967213114754,
|
|
"hit_at_10_wilson_lower": 0.552686960572952,
|
|
"mrr": 0.4176547998525457,
|
|
"n_answerable": 2928,
|
|
"n_unanswerable": 2380,
|
|
"ndcg_at_10": 0.4518849109723208,
|
|
"recall_at_1": 0.34238387978142076,
|
|
"recall_at_10": 0.5654257741347906,
|
|
"recall_at_5": 0.510325591985428,
|
|
"top_hit_from_holder_unanswerable": 0.4042016806722689
|
|
}
|
|
},
|
|
{
|
|
"arm": "fusion_rerank",
|
|
"by_modality": {
|
|
"code": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.6289409862570736,
|
|
"hit_at_10_wilson_lower": 0.6016596375208901,
|
|
"mrr": 0.4450988695127741,
|
|
"n_answerable": 1237,
|
|
"n_unanswerable": 1237,
|
|
"ndcg_at_10": 0.48964513720914205,
|
|
"recall_at_1": 0.35084882780921584,
|
|
"recall_at_10": 0.6289409862570736,
|
|
"recall_at_5": 0.5675020210185934,
|
|
"top_hit_from_holder_unanswerable": 0.6313662085691188
|
|
},
|
|
"conversation": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.5866666666666667,
|
|
"hit_at_10_wilson_lower": 0.5066585675662262,
|
|
"mrr": 0.44420105820105815,
|
|
"n_answerable": 150,
|
|
"n_unanswerable": 150,
|
|
"ndcg_at_10": 0.41993124265105486,
|
|
"recall_at_1": 0.28,
|
|
"recall_at_10": 0.4862222222222222,
|
|
"recall_at_5": 0.42911111111111105,
|
|
"top_hit_from_holder_unanswerable": 0.006666666666666667
|
|
},
|
|
"image": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.7,
|
|
"hit_at_10_wilson_lower": 0.6392449572501318,
|
|
"mrr": 0.5878356481481481,
|
|
"n_answerable": 240,
|
|
"n_unanswerable": 240,
|
|
"ndcg_at_10": 0.6098813565843775,
|
|
"recall_at_1": 0.5208333333333334,
|
|
"recall_at_10": 0.6916666666666667,
|
|
"recall_at_5": 0.6520833333333333,
|
|
"top_hit_from_holder_unanswerable": 0.016666666666666666
|
|
},
|
|
"pdf": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.25326370757180156,
|
|
"hit_at_10_wilson_lower": 0.223750502086736,
|
|
"mrr": 0.1029870695014298,
|
|
"n_answerable": 766,
|
|
"n_unanswerable": 218,
|
|
"ndcg_at_10": 0.13796388574710278,
|
|
"recall_at_1": 0.05221932114882506,
|
|
"recall_at_10": 0.25326370757180156,
|
|
"recall_at_5": 0.16579634464751958,
|
|
"top_hit_from_holder_unanswerable": 0.0871559633027523
|
|
},
|
|
"ui_screenshot": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.3,
|
|
"hit_at_10_wilson_lower": 0.25240562491350604,
|
|
"mrr": 0.18429935515873014,
|
|
"n_answerable": 320,
|
|
"n_unanswerable": 320,
|
|
"ndcg_at_10": 0.21181060150559686,
|
|
"recall_at_1": 0.1375,
|
|
"recall_at_10": 0.3,
|
|
"recall_at_5": 0.25625,
|
|
"top_hit_from_holder_unanswerable": 0.03125
|
|
},
|
|
"video": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.08837209302325581,
|
|
"hit_at_10_wilson_lower": 0.05730442391262212,
|
|
"mrr": 0.04147840531561462,
|
|
"n_answerable": 215,
|
|
"n_unanswerable": 215,
|
|
"ndcg_at_10": 0.0524917978865275,
|
|
"recall_at_1": 0.023255813953488372,
|
|
"recall_at_10": 0.08837209302325581,
|
|
"recall_at_5": 0.06511627906976744,
|
|
"top_hit_from_holder_unanswerable": 0.0
|
|
}
|
|
},
|
|
"latency_seconds": {
|
|
"max": 77.63319287999548,
|
|
"mean": 5.166508504176891,
|
|
"p50": 4.5375522020012795,
|
|
"p95": 9.042528758149817,
|
|
"queries": 5308
|
|
},
|
|
"overall": {
|
|
"abstention_rate_answerable": 0.0,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"hit_at_10": 0.45867486338797814,
|
|
"hit_at_10_wilson_lower": 0.4406921315699621,
|
|
"mrr": 0.3091119405412438,
|
|
"n_answerable": 2928,
|
|
"n_unanswerable": 2380,
|
|
"ndcg_at_10": 0.34146096728664366,
|
|
"recall_at_1": 0.23565573770491804,
|
|
"recall_at_10": 0.4528460837887068,
|
|
"recall_at_5": 0.3913479052823315,
|
|
"top_hit_from_holder_unanswerable": 0.34243697478991597
|
|
}
|
|
}
|
|
],
|
|
"chance": {
|
|
"corpus_sources": 50278,
|
|
"recall_at_1": 1.9889414853415013e-05,
|
|
"recall_at_10": 0.00019889414853415012,
|
|
"recall_at_5": 9.944707426707506e-05
|
|
},
|
|
"live_product": {
|
|
"abstention_rate_answerable": 0.016666666666666666,
|
|
"abstention_rate_unanswerable": 0.0,
|
|
"distinct_sources_in_top10": {
|
|
"max": 10,
|
|
"mean": 5.193333333333333,
|
|
"min": 0
|
|
},
|
|
"hit_at_10": 0.8333333333333334,
|
|
"hit_at_10_wilson_lower": 0.7870033503873167,
|
|
"identical_top10": 0.0,
|
|
"max": 0.3143954080005642,
|
|
"mean": 0.07851241317994816,
|
|
"mrr": 0.6832447089947089,
|
|
"n_answerable": 300,
|
|
"n_unanswerable": 0,
|
|
"ndcg_at_10": 0.7186678703708246,
|
|
"p50": 0.07425041199894622,
|
|
"p95": 0.14573381999798585,
|
|
"queries": 300,
|
|
"recall_at_1": 0.5966666666666667,
|
|
"recall_at_10": 0.8316666666666667,
|
|
"recall_at_5": 0.7816666666666666,
|
|
"top10_overlap_with_bm25_only": 0.13433333333333336,
|
|
"top_hit_from_holder_unanswerable": 0.0,
|
|
"top_k": 10
|
|
},
|
|
"meta": {
|
|
"answerable": 2928,
|
|
"corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347",
|
|
"corpus_sources": 50278,
|
|
"generated_at": "2026-10-08T20:17:12+00:00",
|
|
"git_commit": "dff5977ab638896cc8685c78ee41e5195fc0fbf2",
|
|
"limit_per_modality": null,
|
|
"machine": {
|
|
"cpus": 32,
|
|
"load_average_1_5_15": [
|
|
5.09,
|
|
4.28,
|
|
8.06
|
|
],
|
|
"release": "6.18.40.1-microsoft-standard-WSL2",
|
|
"system": "Linux",
|
|
"threads_per_query": 4
|
|
},
|
|
"models": {
|
|
"dense": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
|
|
"rerank": "BAAI/bge-reranker-base"
|
|
},
|
|
"python": "3.14.4",
|
|
"queries": 5308,
|
|
"sets": {
|
|
"code": "ea7e248386350f95132a398f3710f6ce5f09424ca614fd88c618207f667a553e",
|
|
"conversation": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11",
|
|
"image": "1c74488375c7ec9f3182964d51a5e506273bfd5cf3d126725c3cc431a8333128",
|
|
"pdf": "0cec488e660bf1bfa7ccfc72dbae8b7e3005f0b6a0255028e502296d4ecd4536",
|
|
"ui_screenshot": "2269dd623e67cd851fb3f6b24a4556a8462a37db98e12ad79577c5830167c7c5",
|
|
"video": "4973231d9f16d214b14923d79a34b85c3d2b423624dea752b1ee91ba09ff778c"
|
|
},
|
|
"store_schema_version_at_start": 8
|
|
}
|
|
} |