{
 "meta": {
  "answerable": 2928,
  "extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator",
  "generated_at": "2026-10-07T16:54:23+00:00",
  "git_commit": "c4e2982506faaf350a4b06738923899fc44bdedd",
  "limit_per_modality": null,
  "machine": {
   "cpus": 32,
   "load_average_1_5_15": [
    6.23,
    4.72,
    4.44
   ],
   "release": "6.18.40.1-microsoft-standard-WSL2",
   "system": "Linux"
  },
  "queries": 5308,
  "queries_by_modality": {
   "code": 2474,
   "conversation": 300,
   "image": 480,
   "pdf": 984,
   "ui_screenshot": 640,
   "video": 430
  },
  "sources": 50789,
  "top_k": 10
 },
 "stores": [
  {
   "by_modality": {
    "code": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.952303961196443,
     "hit_at_10_wilson_lower": 0.938963104367135,
     "mrr": 0.6949699092787209,
     "n_answerable": 1237,
     "n_unanswerable": 1237,
     "ndcg_at_10": 0.7581529489950144,
     "recall_at_1": 0.549717057396928,
     "recall_at_10": 0.952303961196443,
     "recall_at_5": 0.8884397736459175,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "conversation": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 1.0,
     "hit_at_10_wilson_lower": 0.9750297556319234,
     "mrr": 0.9271111111111111,
     "n_answerable": 150,
     "n_unanswerable": 150,
     "ndcg_at_10": 0.9366637917237846,
     "recall_at_1": 0.657888888888889,
     "recall_at_10": 0.9927777777777778,
     "recall_at_5": 0.9814444444444445,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "image": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.6541666666666667,
     "hit_at_10_wilson_lower": 0.5919888820581466,
     "mrr": 0.504917328042328,
     "n_answerable": 240,
     "n_unanswerable": 240,
     "ndcg_at_10": 0.5355835725491713,
     "recall_at_1": 0.42291666666666666,
     "recall_at_10": 0.6458333333333334,
     "recall_at_5": 0.6,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "pdf": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.9164490861618799,
     "hit_at_10_wilson_lower": 0.8947140126781276,
     "mrr": 0.691579634464752,
     "n_answerable": 766,
     "n_unanswerable": 218,
     "ndcg_at_10": 0.7454292373597072,
     "recall_at_1": 0.587467362924282,
     "recall_at_10": 0.9164490861618799,
     "recall_at_5": 0.825065274151436,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "ui_screenshot": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.621875,
     "hit_at_10_wilson_lower": 0.567595266832662,
     "mrr": 0.32982886904761904,
     "n_answerable": 320,
     "n_unanswerable": 320,
     "ndcg_at_10": 0.3983431061821887,
     "recall_at_1": 0.21875,
     "recall_at_10": 0.621875,
     "recall_at_5": 0.465625,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "video": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.7162790697674418,
     "hit_at_10_wilson_lower": 0.6526351336615988,
     "mrr": 0.3956736803248431,
     "n_answerable": 215,
     "n_unanswerable": 215,
     "ndcg_at_10": 0.47027044859447076,
     "recall_at_1": 0.2837209302325581,
     "recall_at_10": 0.7162790697674418,
     "recall_at_5": 0.5255813953488372,
     "wrong_holder_confidence_unanswerable": 1.0
    }
   },
   "closed_items": 0,
   "items": 403859,
   "latency_seconds": {
    "max": 15.670968882004672,
    "mean": 0.09444417075096513,
    "p50": 0.08692938649983262,
    "p95": 0.10732038790265505,
    "queries": 5308
   },
   "overall": {
    "abstention_rate_answerable": 0.0,
    "abstention_rate_unanswerable": 0.0,
    "hit_at_10": 0.8674863387978142,
    "hit_at_10_wilson_lower": 0.854722713774439,
    "mrr": 0.6285142358400555,
    "n_answerable": 2928,
    "n_unanswerable": 2380,
    "ndcg_at_10": 0.6852635110385413,
    "recall_at_1": 0.49903802367941713,
    "recall_at_10": 0.8664332877959927,
    "recall_at_5": 0.7801286429872496,
    "wrong_holder_confidence_unanswerable": 1.0
   },
   "store": "mem0_flat"
  },
  {
   "by_modality": {
    "code": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.952303961196443,
     "hit_at_10_wilson_lower": 0.938963104367135,
     "mrr": 0.6949699092787209,
     "n_answerable": 1237,
     "n_unanswerable": 1237,
     "ndcg_at_10": 0.7581529489950144,
     "recall_at_1": 0.549717057396928,
     "recall_at_10": 0.952303961196443,
     "recall_at_5": 0.8884397736459175,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "conversation": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.11333333333333333,
     "hit_at_10": 0.9866666666666667,
     "hit_at_10_wilson_lower": 0.9526930912996325,
     "mrr": 0.9053968253968253,
     "n_answerable": 150,
     "n_unanswerable": 150,
     "ndcg_at_10": 0.9136591750866607,
     "recall_at_1": 0.637888888888889,
     "recall_at_10": 0.9783333333333334,
     "recall_at_5": 0.9458888888888889,
     "wrong_holder_confidence_unanswerable": 0.8866666666666667
    },
    "image": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.5,
     "hit_at_10": 0.6541666666666667,
     "hit_at_10_wilson_lower": 0.5919888820581466,
     "mrr": 0.504917328042328,
     "n_answerable": 240,
     "n_unanswerable": 240,
     "ndcg_at_10": 0.5355835725491713,
     "recall_at_1": 0.42291666666666666,
     "recall_at_10": 0.6458333333333334,
     "recall_at_5": 0.6,
     "wrong_holder_confidence_unanswerable": 0.5
    },
    "pdf": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.9164490861618799,
     "hit_at_10_wilson_lower": 0.8947140126781276,
     "mrr": 0.691579634464752,
     "n_answerable": 766,
     "n_unanswerable": 218,
     "ndcg_at_10": 0.7454292373597072,
     "recall_at_1": 0.587467362924282,
     "recall_at_10": 0.9164490861618799,
     "recall_at_5": 0.825065274151436,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "ui_screenshot": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.621875,
     "hit_at_10_wilson_lower": 0.567595266832662,
     "mrr": 0.32982886904761904,
     "n_answerable": 320,
     "n_unanswerable": 320,
     "ndcg_at_10": 0.3983431061821887,
     "recall_at_1": 0.21875,
     "recall_at_10": 0.621875,
     "recall_at_5": 0.465625,
     "wrong_holder_confidence_unanswerable": 1.0
    },
    "video": {
     "abstention_rate_answerable": 0.0,
     "abstention_rate_unanswerable": 0.0,
     "hit_at_10": 0.7162790697674418,
     "hit_at_10_wilson_lower": 0.6526351336615988,
     "mrr": 0.3956736803248431,
     "n_answerable": 215,
     "n_unanswerable": 215,
     "ndcg_at_10": 0.47027044859447076,
     "recall_at_1": 0.2837209302325581,
     "recall_at_10": 0.7162790697674418,
     "recall_at_5": 0.5255813953488372,
     "wrong_holder_confidence_unanswerable": 1.0
    }
   },
   "closed_items": 27072,
   "items": 403859,
   "latency_seconds": {
    "max": 10.518811015004758,
    "mean": 0.09140969146306362,
    "p50": 0.0847548475030635,
    "p95": 0.10844126994816179,
    "queries": 5308
   },
   "overall": {
    "abstention_rate_answerable": 0.0,
    "abstention_rate_unanswerable": 0.05756302521008403,
    "hit_at_10": 0.8668032786885246,
    "hit_at_10_wilson_lower": 0.8540138593279664,
    "mrr": 0.6274018236620695,
    "n_answerable": 2928,
    "n_unanswerable": 2380,
    "ndcg_at_10": 0.6840849958419674,
    "recall_at_1": 0.4980134335154827,
    "recall_at_10": 0.865693306010929,
    "recall_at_5": 0.7783071493624771,
    "wrong_holder_confidence_unanswerable": 0.942436974789916
   },
   "store": "zep_temporal"
  }
 ]
}