ZacharyZhang-NY
e81923f62e
The 300-probe sample this harness scored by modality read the screenshot modality 0.818 where all 320 of its answerable queries read 0.659, because striding a file-ordered list and truncating at 300 reshuffles the modality mix. Recall is now scored over every answerable query (2,928) and latency over a stratified sample, which is what latency needs and what recall does not. The separate conversation sub-pass is gone: conversation is inside the suite, and by_category already carries its six LongMemEval types.
156 lines
7.4 KiB
Python
156 lines
7.4 KiB
Python
"""The LEGS page: one markdown table per question the harness was built to answer."""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def write_markdown(report: dict[str, Any], path: Path) -> None:
|
|
machine = report["machine"]
|
|
header = [
|
|
"| leg configuration | atoms asked | recall@10 | MRR | hit@10 | 95% lower | newest first | p50 ms | p95 ms | distinct sources |",
|
|
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
|
|
]
|
|
rows = []
|
|
for arm in report["arms"]:
|
|
newest = arm["newest_gold_first"]
|
|
rows.append(
|
|
f"| {arm['label']} | {arm['top_k']} | {arm['recall_at_10']:.3f} | {arm['mrr']:.3f} | "
|
|
f"{arm['hit_at_10']:.3f} | {arm['wilson_lower']:.3f} | "
|
|
f"{'—' if newest is None else f'{newest:.2f}'} | {arm['p50_seconds'] * 1000:.1f} | "
|
|
f"{arm['p95_seconds'] * 1000:.1f} | {arm['median_distinct_sources']} |"
|
|
)
|
|
modalities = sorted({modality for arm in report["arms"] for modality in arm["by_modality"]})
|
|
modality_rows = [
|
|
"| leg configuration | " + " | ".join(modalities) + " |",
|
|
"| --- |" + " --- |" * len(modalities),
|
|
]
|
|
for arm in report["arms"]:
|
|
modality_rows.append(
|
|
f"| {arm['label']} | "
|
|
+ " | ".join(f"{arm['by_modality'].get(modality, float('nan')):.3f}" for modality in modalities)
|
|
+ " |"
|
|
)
|
|
counts = report["arms"][0]["modality_counts"]
|
|
|
|
def category_table(title: str, field: str, sizes_field: str) -> list[str]:
|
|
"""A table over the question types the set records; unlabelled sets stay out of it."""
|
|
rows = report["arms"]
|
|
names = sorted({name for row in rows for name in row[field] if name != "unlabelled"})
|
|
if not names:
|
|
return []
|
|
lines = [
|
|
f"## {title}",
|
|
"",
|
|
"| leg configuration | " + " | ".join(names) + " |",
|
|
"| --- |" + " --- |" * len(names),
|
|
]
|
|
for arm, row in zip(report["arms"], rows, strict=True):
|
|
values, sizes = row[field], row[sizes_field]
|
|
cells = []
|
|
for name in names:
|
|
if name not in values:
|
|
cells.append("—")
|
|
elif field == "by_category":
|
|
cells.append(f"{values[name]:.3f}")
|
|
else:
|
|
cells.append(f"{values[name]:.2f} ({sizes[name]})")
|
|
lines.append(f"| {arm['label']} | " + " | ".join(cells) + " |")
|
|
sizes = rows[0][sizes_field]
|
|
lines += [
|
|
"",
|
|
"_Queries per type: "
|
|
+ ", ".join(f"{name} {size}" for name, size in sizes.items() if name != "unlabelled")
|
|
+ "._",
|
|
]
|
|
return lines
|
|
path.write_text(
|
|
"\n".join(
|
|
[
|
|
"# LEGS: what each retrieval leg is worth",
|
|
"",
|
|
(
|
|
f"- commit: `{report['commit']}`, schema v{report['schema_version']} — the copy "
|
|
f"arrives at v{report['source_schema_version']} and the product migrates it when "
|
|
f"it opens, so every arm ran on v{report['schema_version']}"
|
|
),
|
|
(
|
|
f"- store: {report['store']} — a copy of {report['source_store']}, which carries "
|
|
f"{report['dense_vectors']:,} dense vectors built by bench/dense_scale. The build "
|
|
"sits outside the measured latency, and the frozen baseline itself holds no "
|
|
"vectors, so a dense arm there would rank an empty index."
|
|
),
|
|
(
|
|
f"- {report['queries']:,} answerable queries scored whole, each asked as the "
|
|
f"holder the frozen set assigns it; latency is timed on a stratified sample of "
|
|
f"{report['timed_queries']}, because a sample sized for timing is not sized for a "
|
|
"modality's recall — the 300-probe sample this harness used before read the "
|
|
"screenshot modality 0.818 where all 320 of its answerable queries read 0.659"
|
|
),
|
|
*(
|
|
[]
|
|
if report.get("picture_index") is None
|
|
else [
|
|
(
|
|
f"- picture index: {report['picture_index']['atoms']:,} atoms at "
|
|
f"{report['picture_index']['dim']} dimensions, "
|
|
f"{report['picture_index']['bytes']:,} bytes, read from "
|
|
f"{report['picture_index']['path']}"
|
|
),
|
|
(
|
|
f"- encoder speed: {report['encode_rate']['per_second']:.1f} pictures/s "
|
|
f"measured on {report['encode_rate']['pictures']} pictures, so the "
|
|
f"{report['picture_index']['atoms']:,} pictures in that index cost about "
|
|
f"{report['picture_index']['atoms'] / report['encode_rate']['per_second'] / 60:.1f} "
|
|
"minutes to encode"
|
|
),
|
|
]
|
|
),
|
|
(
|
|
f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, "
|
|
f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"
|
|
),
|
|
"",
|
|
"Recall is scored on the first ten **sources** after atoms are collapsed to their source",
|
|
"by first hit, which is how every other arm in this repo is scored.",
|
|
"",
|
|
*header,
|
|
*rows,
|
|
"",
|
|
"## recall@10 by modality",
|
|
"",
|
|
*modality_rows,
|
|
"",
|
|
f"_Query counts per modality: {', '.join(f'{name} {count}' for name, count in counts.items())}._",
|
|
"",
|
|
*category_table("recall@10 by question type", "by_category", "category_counts"),
|
|
"",
|
|
*category_table(
|
|
"the newest of the gold memories came back first",
|
|
"newest_gold_first_by_category",
|
|
"newest_gold_first_counts",
|
|
),
|
|
"",
|
|
(
|
|
"recall@10 counts any gold memory, so a knowledge-update question can score 1.000 "
|
|
"while the ranking still leads with the value the speaker has since replaced. The "
|
|
"table above asks the sharper question: when the answer lives in several dated "
|
|
"memories, is the newest one the first one returned? Only questions with at least two "
|
|
"dated gold memories and at least one returned are counted."
|
|
),
|
|
"",
|
|
(
|
|
"The picture leg is not a switch this harness flips per row: the product runs it only "
|
|
"when the sparse page comes back thin, so these rows measure the product with the leg "
|
|
"available, not a forced always-on fusion."
|
|
),
|
|
"",
|
|
(
|
|
"_Latency is wall time per search on the machine named above with the thread budget "
|
|
"the benchmark pins; a run on a loaded machine is not comparable._"
|
|
),
|
|
"",
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|