ZacharyZhang-NY
77219c930b
Ground truth comes only from third-party annotations (MMLongBench-Doc
evidence_page_ids, VisRAG qrels, ScreenSpot-v2 bboxes, Video-MME QA,
LongMemEval answer_session_ids, CodeSearchNet func_path+func_name) and
from document structure; no model judgement is used anywhere.
- bench/corpus/manifest.jsonl: 28 sources with url, retrieved_at, sha256,
annotation_source, license, files[]; verified by bench.manifest_check
- bench/sets/{pdf,image,ui_screenshot,video,code,conversation}.{jsonl} plus
corpus indexes and a manifest that freezes every file's sha256
- 2,928 answerable + 2,380 unanswerable queries over 854 holders; every
query carries scope {holder, scenario, as_of} for transactive routing
- bench/validate_sets.py fails on missing gold, holder mismatch between
gold and query, gold observed after as_of, and manifest digest drift
106 lines
4.3 KiB
Python
106 lines
4.3 KiB
Python
"""Run every set builder in one pass and freeze what the whole suite contains.
|
|
|
|
Each builder writes its own corpus, queries, and report; this orchestrator only
|
|
adds the suite-level manifest that pins the corpus snapshot and the digest of
|
|
every generated file, so a later run can prove it is measuring the same data.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import json
|
|
|
|
from bench.sets_common import CORPUS, MODALITIES, SETS, read_jsonl, sha256_file
|
|
|
|
BUILDERS = {
|
|
"pdf": "bench.build_mmlongbench",
|
|
"image": "bench.build_visrag",
|
|
"ui_screenshot": "bench.build_screenspot",
|
|
"video": "bench.build_videomme",
|
|
"code": "bench.build_code",
|
|
"conversation": "bench.build_longmemeval",
|
|
}
|
|
|
|
if set(BUILDERS) != set(MODALITIES):
|
|
raise SystemExit(
|
|
f"builders cover {sorted(BUILDERS)} but the schema demands {sorted(MODALITIES)}"
|
|
)
|
|
|
|
|
|
def read_report(modality: str) -> dict:
|
|
"""A builder that produced no report is a broken build, not a missing optional file."""
|
|
path = SETS / "reports" / f"{modality}.json"
|
|
try:
|
|
loaded = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise SystemExit(f"{path}: unreadable builder report: {error}") from error
|
|
if not isinstance(loaded, dict):
|
|
raise SystemExit(f"{path}: expected a json object")
|
|
return loaded
|
|
|
|
|
|
def main() -> None:
|
|
for modality in MODALITIES:
|
|
print(f"== {modality}")
|
|
importlib.import_module(BUILDERS[modality]).main()
|
|
|
|
modalities: dict[str, dict] = {}
|
|
for modality in MODALITIES:
|
|
queries = list(read_jsonl(SETS / f"{modality}.jsonl"))
|
|
report = read_report(modality)
|
|
if report.get("queries") != len(queries):
|
|
raise SystemExit(
|
|
f"{modality}: report says {report.get('queries')} queries, "
|
|
f"file holds {len(queries)}"
|
|
)
|
|
datasets = report.get("datasets")
|
|
holders = report.get("holders")
|
|
if not isinstance(datasets, list) or not all(isinstance(d, str) for d in datasets):
|
|
raise SystemExit(f"{modality}: report must name its upstream datasets, got {datasets!r}")
|
|
if not isinstance(holders, list) or not all(isinstance(h, str) for h in holders):
|
|
raise SystemExit(f"{modality}: report must name its holders, got {holders!r}")
|
|
modalities[modality] = {
|
|
"corpus_file": f"corpus/{modality}.jsonl",
|
|
"corpus_records": sum(1 for _ in read_jsonl(SETS / "corpus" / f"{modality}.jsonl")),
|
|
"corpus_sha256": sha256_file(SETS / "corpus" / f"{modality}.jsonl"),
|
|
"queries_file": f"{modality}.jsonl",
|
|
"queries": len(queries),
|
|
"queries_answerable": sum(1 for query in queries if not query["unanswerable"]),
|
|
"queries_unanswerable": sum(1 for query in queries if query["unanswerable"]),
|
|
"queries_sha256": sha256_file(SETS / f"{modality}.jsonl"),
|
|
"holders": holders,
|
|
"sources": datasets,
|
|
}
|
|
|
|
manifest = {
|
|
"corpus_manifest": "corpus/manifest.jsonl",
|
|
"corpus_manifest_sha256": sha256_file(CORPUS / "manifest.jsonl"),
|
|
"modalities": modalities,
|
|
"corpus_records": sum(item["corpus_records"] for item in modalities.values()),
|
|
"queries": sum(item["queries"] for item in modalities.values()),
|
|
"queries_answerable": sum(item["queries_answerable"] for item in modalities.values()),
|
|
"queries_unanswerable": sum(item["queries_unanswerable"] for item in modalities.values()),
|
|
"holders": sum(len(item["holders"]) for item in modalities.values()),
|
|
}
|
|
path = SETS / "manifest.json"
|
|
path.write_text(json.dumps(manifest, indent=1, sort_keys=True), encoding="utf-8")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"corpus_records": manifest["corpus_records"],
|
|
"queries": manifest["queries"],
|
|
"queries_answerable": manifest["queries_answerable"],
|
|
"queries_unanswerable": manifest["queries_unanswerable"],
|
|
"holders": manifest["holders"],
|
|
"corpus_manifest_sha256": manifest["corpus_manifest_sha256"][:16],
|
|
"manifest": str(path),
|
|
},
|
|
indent=1,
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|