ZacharyZhang-NY
deabbf76b8
The tier report is regenerated against a52c104, which carries the harness
that produced it, and now names the schema the product actually ran on: the
copy arrives at v9, the product migrates it on open, every arm ran on v10.
Every recall cell is identical across three runs on three different machine
loads (0.847 / 0.863 / 0.873 / 0.860 / 0.873 / 0.817 / 0.848), which is the
regression proof p2-3 needs: the tier code adds state and rules, not a
ranker, and this corpus holds no conflicts and no archived rows. Latency is
not comparable across those runs (p50 101.6 / 87.5 / 122.5 ms at load 27.4 /
21.4 / 17.2) and the report says so.
The memory subset is now measured whole — all 150 answerable conversation
queries instead of 15 sampled probes — because a 15-probe sample leaves one
to three queries per LongMemEval question type. It gives the two categories
the contract asks for: knowledge-update 1.000 and temporal-reasoning 0.879
as shipped, 0.950 with the dense leg. The newest-first column is the sharper
question beside them: 0.52 / 0.48 / 0.37 as shipped, so a perfect recall can
still lead with a value the speaker has since replaced.
CHECKLIST p2-3 is marked with the measured cells, the two real defects the
tests caught (a consolidation watermark that double-counted under a lagging
clock, and typer calling a variadic command callback by keyword), and the
deviation that decay is not a stored field because an unmeasured decay
formula would be a story.
1015 lines
37 KiB
JSON
1015 lines
37 KiB
JSON
{
|
|
"commit": "a52c10499b48f171db1ad40f20141acb34028200",
|
|
"schema_version": 10,
|
|
"source_schema_version": 9,
|
|
"store": "bench/runs/leg_check/data/ely_eye.sqlite",
|
|
"source_store": "bench/runs/dense_scale",
|
|
"dense_vectors": 153857,
|
|
"picture_index": {
|
|
"path": "bench/runs/leg_check_index",
|
|
"atoms": 53668,
|
|
"dim": 512,
|
|
"bytes": 109912192
|
|
},
|
|
"encode_rate": {
|
|
"pictures": 200,
|
|
"seconds": 10.621729820995824,
|
|
"per_second": 18.829324730578517
|
|
},
|
|
"probes": 300,
|
|
"memory_queries": 150,
|
|
"memory_modality": "conversation",
|
|
"arms": [
|
|
{
|
|
"label": "words only",
|
|
"top_k": 10,
|
|
"dense": false,
|
|
"visual": false,
|
|
"recall_at_10": 0.8466666666666667,
|
|
"mrr": 0.6781997354497354,
|
|
"hit_at_10": 0.8466666666666667,
|
|
"wilson_lower": 0.8015339429418983,
|
|
"p50_seconds": 0.08283499400567962,
|
|
"p95_seconds": 0.15074783855670837,
|
|
"median_distinct_sources": 7.0,
|
|
"by_modality": {
|
|
"code": 0.9448818897637795,
|
|
"conversation": 1.0,
|
|
"image": 0.72,
|
|
"pdf": 0.8333333333333334,
|
|
"ui_screenshot": 0.8181818181818182,
|
|
"video": 0.4090909090909091
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 0.0,
|
|
"Film & Television/Action Reasoning/long": 0.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 0.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 0.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 0.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 1.0,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 1.0,
|
|
"unlabelled": 0.8745247148288974
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.3333333333333333,
|
|
"multi-session": 0.5,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.9158888888888889,
|
|
"mrr": 0.8230000000000001,
|
|
"hit_at_10": 0.9533333333333334,
|
|
"wilson_lower": 0.9068135276008726,
|
|
"by_modality": {
|
|
"conversation": 0.9158888888888889
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.8566666666666667,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.8,
|
|
"single-session-user": 0.96,
|
|
"temporal-reasoning": 0.8786666666666666
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.45588235294117646,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.52,
|
|
"multi-session": 0.4583333333333333,
|
|
"temporal-reasoning": 0.3684210526315789
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 24,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"label": "words only (the configured 24)",
|
|
"top_k": 24,
|
|
"dense": false,
|
|
"visual": false,
|
|
"recall_at_10": 0.8633333333333333,
|
|
"mrr": 0.6823932886285827,
|
|
"hit_at_10": 0.8633333333333333,
|
|
"wilson_lower": 0.8198445870022494,
|
|
"p50_seconds": 0.12253823449282208,
|
|
"p95_seconds": 0.22357540469092782,
|
|
"median_distinct_sources": 8.0,
|
|
"by_modality": {
|
|
"code": 0.9448818897637795,
|
|
"conversation": 1.0,
|
|
"image": 0.72,
|
|
"pdf": 0.8846153846153846,
|
|
"ui_screenshot": 0.8181818181818182,
|
|
"video": 0.45454545454545453
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 0.0,
|
|
"Film & Television/Action Reasoning/long": 0.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 0.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 1.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 0.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 1.0,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 1.0,
|
|
"unlabelled": 0.8897338403041825
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.3333333333333333,
|
|
"multi-session": 0.5,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.9158888888888889,
|
|
"mrr": 0.825520590023787,
|
|
"hit_at_10": 0.9533333333333334,
|
|
"wilson_lower": 0.9068135276008726,
|
|
"by_modality": {
|
|
"conversation": 0.9158888888888889
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.8566666666666667,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.8,
|
|
"single-session-user": 0.96,
|
|
"temporal-reasoning": 0.8786666666666666
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.463768115942029,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.52,
|
|
"multi-session": 0.48,
|
|
"temporal-reasoning": 0.3684210526315789
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"label": "words only, wider page",
|
|
"top_k": 30,
|
|
"dense": false,
|
|
"visual": false,
|
|
"recall_at_10": 0.8733333333333333,
|
|
"mrr": 0.6856373362476303,
|
|
"hit_at_10": 0.8733333333333333,
|
|
"wilson_lower": 0.8309188299507383,
|
|
"p50_seconds": 0.08374732300580945,
|
|
"p95_seconds": 0.157951294999657,
|
|
"median_distinct_sources": 8.0,
|
|
"by_modality": {
|
|
"code": 0.9448818897637795,
|
|
"conversation": 1.0,
|
|
"image": 0.72,
|
|
"pdf": 0.9102564102564102,
|
|
"ui_screenshot": 0.8181818181818182,
|
|
"video": 0.5
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 0.0,
|
|
"Film & Television/Action Reasoning/long": 0.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 0.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 1.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 1.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 1.0,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 1.0,
|
|
"unlabelled": 0.8973384030418251
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.3333333333333333,
|
|
"multi-session": 0.5,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.9158888888888889,
|
|
"mrr": 0.8257770002801973,
|
|
"hit_at_10": 0.9533333333333334,
|
|
"wilson_lower": 0.9068135276008726,
|
|
"by_modality": {
|
|
"conversation": 0.9158888888888889
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.8566666666666667,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.8,
|
|
"single-session-user": 0.96,
|
|
"temporal-reasoning": 0.8786666666666666
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.463768115942029,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.52,
|
|
"multi-session": 0.48,
|
|
"temporal-reasoning": 0.3684210526315789
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"label": "+ dense leg",
|
|
"top_k": 10,
|
|
"dense": true,
|
|
"visual": false,
|
|
"recall_at_10": 0.86,
|
|
"mrr": 0.6686600529100529,
|
|
"hit_at_10": 0.86,
|
|
"wilson_lower": 0.8161683876490509,
|
|
"p50_seconds": 0.7133175364942872,
|
|
"p95_seconds": 0.8329682753508678,
|
|
"median_distinct_sources": 7.0,
|
|
"by_modality": {
|
|
"code": 0.9448818897637795,
|
|
"conversation": 1.0,
|
|
"image": 0.68,
|
|
"pdf": 0.8717948717948718,
|
|
"ui_screenshot": 0.8484848484848485,
|
|
"video": 0.45454545454545453
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 0.0,
|
|
"Film & Television/Action Reasoning/long": 1.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 0.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 0.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 0.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 1.0,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 1.0,
|
|
"unlabelled": 0.8859315589353612
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.6666666666666666,
|
|
"multi-session": 0.0,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.965,
|
|
"mrr": 0.8735925925925926,
|
|
"hit_at_10": 0.98,
|
|
"wilson_lower": 0.9428533167296139,
|
|
"by_modality": {
|
|
"conversation": 0.965
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.92,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.96,
|
|
"single-session-user": 0.96,
|
|
"temporal-reasoning": 0.95
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.4411764705882353,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.4,
|
|
"multi-session": 0.4583333333333333,
|
|
"temporal-reasoning": 0.47368421052631576
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 24,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"label": "+ dense leg, wider page",
|
|
"top_k": 30,
|
|
"dense": true,
|
|
"visual": false,
|
|
"recall_at_10": 0.8733333333333333,
|
|
"mrr": 0.6752862924112923,
|
|
"hit_at_10": 0.8733333333333333,
|
|
"wilson_lower": 0.8309188299507383,
|
|
"p50_seconds": 0.5790213440050138,
|
|
"p95_seconds": 0.7566172773556901,
|
|
"median_distinct_sources": 9.0,
|
|
"by_modality": {
|
|
"code": 0.9448818897637795,
|
|
"conversation": 1.0,
|
|
"image": 0.68,
|
|
"pdf": 0.9102564102564102,
|
|
"ui_screenshot": 0.8484848484848485,
|
|
"video": 0.5
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 0.0,
|
|
"Film & Television/Action Reasoning/long": 1.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 0.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 0.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 1.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 1.0,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 1.0,
|
|
"unlabelled": 0.8973384030418251
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.6666666666666666,
|
|
"multi-session": 0.0,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.965,
|
|
"mrr": 0.8749183501683501,
|
|
"hit_at_10": 0.98,
|
|
"wilson_lower": 0.9428533167296139,
|
|
"by_modality": {
|
|
"conversation": 0.965
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.92,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.96,
|
|
"single-session-user": 0.96,
|
|
"temporal-reasoning": 0.95
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.43478260869565216,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.4,
|
|
"multi-session": 0.44,
|
|
"temporal-reasoning": 0.47368421052631576
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"label": "+ picture leg",
|
|
"top_k": 10,
|
|
"dense": false,
|
|
"visual": true,
|
|
"recall_at_10": 0.8166666666666667,
|
|
"mrr": 0.5646732804232805,
|
|
"hit_at_10": 0.8166666666666667,
|
|
"wilson_lower": 0.7689713823226987,
|
|
"p50_seconds": 0.14521063199208584,
|
|
"p95_seconds": 0.3376805504958611,
|
|
"median_distinct_sources": 9.0,
|
|
"by_modality": {
|
|
"code": 0.889763779527559,
|
|
"conversation": 1.0,
|
|
"image": 0.72,
|
|
"pdf": 0.8333333333333334,
|
|
"ui_screenshot": 0.7272727272727273,
|
|
"video": 0.45454545454545453
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1.0,
|
|
"Film & Television/Action Reasoning/long": 0.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 0.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 0.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 0.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 1.0,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 1.0,
|
|
"unlabelled": 0.8365019011406845
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.3333333333333333,
|
|
"multi-session": 0.5,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.9158888888888889,
|
|
"mrr": 0.8230000000000001,
|
|
"hit_at_10": 0.9533333333333334,
|
|
"wilson_lower": 0.9068135276008726,
|
|
"by_modality": {
|
|
"conversation": 0.9158888888888889
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.8566666666666667,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.8,
|
|
"single-session-user": 0.96,
|
|
"temporal-reasoning": 0.8786666666666666
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.45588235294117646,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.52,
|
|
"multi-session": 0.4583333333333333,
|
|
"temporal-reasoning": 0.3684210526315789
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 24,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"label": "+ picture leg, wider page",
|
|
"top_k": 30,
|
|
"dense": false,
|
|
"visual": true,
|
|
"recall_at_10": 0.8476666666666667,
|
|
"mrr": 0.5375587247956174,
|
|
"hit_at_10": 0.85,
|
|
"wilson_lower": 0.8051823888546709,
|
|
"p50_seconds": 0.21949212049366906,
|
|
"p95_seconds": 0.36040221269940964,
|
|
"median_distinct_sources": 22.0,
|
|
"by_modality": {
|
|
"code": 0.889763779527559,
|
|
"conversation": 0.9533333333333334,
|
|
"image": 0.72,
|
|
"pdf": 0.8974358974358975,
|
|
"ui_screenshot": 0.7878787878787878,
|
|
"video": 0.5909090909090909
|
|
},
|
|
"modality_counts": {
|
|
"code": 127,
|
|
"conversation": 15,
|
|
"image": 25,
|
|
"pdf": 78,
|
|
"ui_screenshot": 33,
|
|
"video": 22
|
|
},
|
|
"by_category": {
|
|
"Artistic Performance/Counting Problem/long": 1.0,
|
|
"Artistic Performance/Counting Problem/short": 0.0,
|
|
"Artistic Performance/Object Recognition/short": 0.0,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1.0,
|
|
"Film & Television/Action Reasoning/long": 0.0,
|
|
"Film & Television/Action Reasoning/medium": 0.0,
|
|
"Film & Television/Object Recognition/short": 0.0,
|
|
"Film & Television/Spatial Perception/short": 1.0,
|
|
"Knowledge/Action Recognition/medium": 1.0,
|
|
"Knowledge/Attribute Perception/short": 0.0,
|
|
"Knowledge/Information Synopsis/short": 1.0,
|
|
"Knowledge/Object Reasoning/long": 1.0,
|
|
"Knowledge/Object Reasoning/medium": 1.0,
|
|
"Life Record/Action Recognition/short": 1.0,
|
|
"Life Record/Object Recognition/medium": 0.0,
|
|
"Life Record/Object Recognition/short": 1.0,
|
|
"Life Record/Temporal Reasoning/long": 1.0,
|
|
"Multilingual/Object Recognition/long": 1.0,
|
|
"Multilingual/Object Recognition/short": 0.0,
|
|
"Sports Competition/Information Synopsis/short": 1.0,
|
|
"Sports Competition/Object Recognition/short": 1.0,
|
|
"knowledge-update": 1.0,
|
|
"multi-session": 0.75,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 1.0,
|
|
"single-session-user": 1.0,
|
|
"temporal-reasoning": 0.9,
|
|
"unlabelled": 0.8631178707224335
|
|
},
|
|
"category_counts": {
|
|
"Artistic Performance/Counting Problem/long": 1,
|
|
"Artistic Performance/Counting Problem/short": 2,
|
|
"Artistic Performance/Object Recognition/short": 1,
|
|
"Artistic Performance/Temporal Reasoning/medium": 1,
|
|
"Film & Television/Action Reasoning/long": 1,
|
|
"Film & Television/Action Reasoning/medium": 1,
|
|
"Film & Television/Object Recognition/short": 1,
|
|
"Film & Television/Spatial Perception/short": 1,
|
|
"Knowledge/Action Recognition/medium": 1,
|
|
"Knowledge/Attribute Perception/short": 1,
|
|
"Knowledge/Information Synopsis/short": 1,
|
|
"Knowledge/Object Reasoning/long": 1,
|
|
"Knowledge/Object Reasoning/medium": 1,
|
|
"Life Record/Action Recognition/short": 1,
|
|
"Life Record/Object Recognition/medium": 1,
|
|
"Life Record/Object Recognition/short": 1,
|
|
"Life Record/Temporal Reasoning/long": 1,
|
|
"Multilingual/Object Recognition/long": 1,
|
|
"Multilingual/Object Recognition/short": 1,
|
|
"Sports Competition/Information Synopsis/short": 1,
|
|
"Sports Competition/Object Recognition/short": 1,
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"single-session-assistant": 3,
|
|
"single-session-preference": 2,
|
|
"single-session-user": 3,
|
|
"temporal-reasoning": 2,
|
|
"unlabelled": 263
|
|
},
|
|
"newest_gold_first": 0.42857142857142855,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.3333333333333333,
|
|
"multi-session": 0.5,
|
|
"temporal-reasoning": 0.5
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 3,
|
|
"multi-session": 2,
|
|
"temporal-reasoning": 2
|
|
},
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.8662222222222222,
|
|
"mrr": 0.5736806113749848,
|
|
"hit_at_10": 0.92,
|
|
"wilson_lower": 0.8653786362494686,
|
|
"by_modality": {
|
|
"conversation": 0.8662222222222222
|
|
},
|
|
"modality_counts": {
|
|
"conversation": 150
|
|
},
|
|
"by_category": {
|
|
"knowledge-update": 0.96,
|
|
"multi-session": 0.7666666666666667,
|
|
"single-session-assistant": 1.0,
|
|
"single-session-preference": 0.72,
|
|
"single-session-user": 0.92,
|
|
"temporal-reasoning": 0.8306666666666667
|
|
},
|
|
"category_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"newest_gold_first": 0.45588235294117646,
|
|
"newest_gold_first_by_category": {
|
|
"knowledge-update": 0.52,
|
|
"multi-session": 0.4583333333333333,
|
|
"temporal-reasoning": 0.3684210526315789
|
|
},
|
|
"newest_gold_first_counts": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 24,
|
|
"temporal-reasoning": 19
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"machine": {
|
|
"system": "Linux",
|
|
"release": "6.18.40.1-microsoft-standard-WSL2",
|
|
"cpus": 32,
|
|
"threads_per_query": 4,
|
|
"load_average_1_5_15": [
|
|
17.23,
|
|
10.83,
|
|
6.53
|
|
]
|
|
}
|
|
} |