ZacharyZhang-NY
77219c930b
Ground truth comes only from third-party annotations (MMLongBench-Doc
evidence_page_ids, VisRAG qrels, ScreenSpot-v2 bboxes, Video-MME QA,
LongMemEval answer_session_ids, CodeSearchNet func_path+func_name) and
from document structure; no model judgement is used anywhere.
- bench/corpus/manifest.jsonl: 28 sources with url, retrieved_at, sha256,
annotation_source, license, files[]; verified by bench.manifest_check
- bench/sets/{pdf,image,ui_screenshot,video,code,conversation}.{jsonl} plus
corpus indexes and a manifest that freezes every file's sha256
- 2,928 answerable + 2,380 unanswerable queries over 854 holders; every
query carries scope {holder, scenario, as_of} for transactive routing
- bench/validate_sets.py fails on missing gold, holder mismatch between
gold and query, gold observed after as_of, and manifest digest drift
178 lines
3.0 KiB
JSON
178 lines
3.0 KiB
JSON
{
|
|
"as_of_pushed_to_newest_answer_session": 10,
|
|
"by_category": {
|
|
"knowledge-update": 25,
|
|
"multi-session": 25,
|
|
"single-session-assistant": 25,
|
|
"single-session-preference": 25,
|
|
"single-session-user": 25,
|
|
"temporal-reasoning": 25
|
|
},
|
|
"corpus_records": 7188,
|
|
"corpus_sha256": "9cc8263c6a6d54ddc86ff1936ca57ddfd03e58d4b6dc0013cd0c47100e95a92f",
|
|
"datasets": [
|
|
"xiaowu0162/longmemeval-cleaned"
|
|
],
|
|
"holders": [
|
|
"00ca467f",
|
|
"01493427",
|
|
"06878be2",
|
|
"06db6396",
|
|
"06f04340",
|
|
"07b6f563",
|
|
"0862e8bf",
|
|
"08e075c7",
|
|
"08f4fc43",
|
|
"09d032c9",
|
|
"0a34ad58",
|
|
"0a995998",
|
|
"0ddfec37_abs",
|
|
"0e5e2d1a",
|
|
"0edc2aef",
|
|
"10e09553",
|
|
"118b2229",
|
|
"15745da0_abs",
|
|
"157a136e",
|
|
"16c90bf4",
|
|
"1903aded",
|
|
"195a1a1b",
|
|
"19b5f2b3_abs",
|
|
"1a1907b4",
|
|
"1c0ddc50",
|
|
"1d4e3b97",
|
|
"1da05512",
|
|
"1de5cff2",
|
|
"2133c1b5",
|
|
"21436231",
|
|
"2311e44b",
|
|
"2698e78f",
|
|
"28dc39ac",
|
|
"29f2956b",
|
|
"2a1811e2",
|
|
"2bf43736",
|
|
"311778f1",
|
|
"32260d93",
|
|
"352ab8bd",
|
|
"35a27287",
|
|
"37d43f65",
|
|
"37f165cf",
|
|
"38146c39",
|
|
"3ba21379",
|
|
"3d86fd0a",
|
|
"4100d0a0",
|
|
"41275add",
|
|
"45dc21b6",
|
|
"46a3abf7",
|
|
"488d3006",
|
|
"4c36ccef",
|
|
"4dfccbf7",
|
|
"505af2f5",
|
|
"50635ada",
|
|
"51a45a95",
|
|
"51c32626",
|
|
"54026fce",
|
|
"561fabcd",
|
|
"577d4d32",
|
|
"57f827a0",
|
|
"58470ed2",
|
|
"5a4f22c0",
|
|
"5d3d2817",
|
|
"60472f9c",
|
|
"6071bd76",
|
|
"6456829e_abs",
|
|
"65240037",
|
|
"66f24dbb",
|
|
"6a1eabeb",
|
|
"6aeb4375_abs",
|
|
"6b168ec8",
|
|
"6b7dfb22",
|
|
"6d550036",
|
|
"6e984302",
|
|
"70b3e69b",
|
|
"71a3fd6b",
|
|
"726462e0",
|
|
"7401057b",
|
|
"75499fd8",
|
|
"75832dbd",
|
|
"75f70248",
|
|
"77eafa52",
|
|
"7a8d0b71",
|
|
"830ce83f",
|
|
"8464fc84",
|
|
"853b0a1d",
|
|
"86b68151",
|
|
"87f22b4a",
|
|
"89527b6b",
|
|
"89941a94",
|
|
"8a137a7f",
|
|
"8a2466db",
|
|
"8b9d4367",
|
|
"8e91e7d9",
|
|
"8ebdbe50",
|
|
"95228167",
|
|
"95bcc1c8",
|
|
"982b5123",
|
|
"9bbe84a2",
|
|
"a08a253f",
|
|
"a2f3aa27",
|
|
"a3045048",
|
|
"a40e080f",
|
|
"a4996e51",
|
|
"a82c026e",
|
|
"a89d7624",
|
|
"af8d2e46",
|
|
"afdc33df",
|
|
"b0479f84",
|
|
"b3c15d39",
|
|
"b46e15ee",
|
|
"b6019101",
|
|
"b759caee",
|
|
"b86304ba",
|
|
"bc149d6b",
|
|
"c6853660",
|
|
"c7cf7dfd",
|
|
"c8090214_abs",
|
|
"cc06de0d",
|
|
"cc539528",
|
|
"cf22b7bf",
|
|
"d596882b",
|
|
"d905b33f",
|
|
"db467c8c",
|
|
"e3fc4d6e",
|
|
"e4e14d04",
|
|
"e5ba910e_abs",
|
|
"e61a7584",
|
|
"e8a79c70",
|
|
"ed4ddc30",
|
|
"eeda8a6d",
|
|
"f0e564bc",
|
|
"gpt4_0b2f1d21",
|
|
"gpt4_1d80365e",
|
|
"gpt4_2312f94c",
|
|
"gpt4_2f56ae70",
|
|
"gpt4_2f8be40d",
|
|
"gpt4_468eb064",
|
|
"gpt4_4ef30696",
|
|
"gpt4_5501fe77",
|
|
"gpt4_5dcc0aab",
|
|
"gpt4_6dc9b45b",
|
|
"gpt4_78cf46a3",
|
|
"gpt4_7ddcf75f",
|
|
"gpt4_85da3956",
|
|
"gpt4_93159ced_abs",
|
|
"gpt4_a2d1d1f6",
|
|
"gpt4_b5700ca9",
|
|
"gpt4_d6585ce9",
|
|
"gpt4_e414231e"
|
|
],
|
|
"items_dropped_duplicate_sessions": 13,
|
|
"items_sampled": 150,
|
|
"items_total": 500,
|
|
"queries": 300,
|
|
"queries_answerable": 150,
|
|
"queries_cross_holder_unanswerable": 150,
|
|
"queries_sha256": "dc106d90e7749bee8be01b1fab3b645a5459d7527ef40e7fbf983677d5898d11",
|
|
"queries_with_turn_level_gold": 136,
|
|
"revision": "98d7416c24c778c2fee6e6f3006e7a073259d48f",
|
|
"sessions": 7188
|
|
} |