Time every modality and ask which memory came back first
One stride over file order, truncated to the probe count, dropped the tail of the suite: a 300-probe sample held 128 code, 86 pdf, 36 screenshot, 26 image and 24 video queries and no conversation query at all, so every report built from it measured five of six modalities. The budget is now split by suite weight with a floor of one probe per modality, and a test pins that the frozen sample covers all six. The set's question type rides along on each query, and a second metric asks the question recall@10 cannot: when the answer lives in several dated memories, did the newest come back first? A knowledge-update question scores 1.000 on recall while the ranking still leads with the value the speaker has since replaced, and that is the failure the next task is meant to fix.
This commit is contained in:
1 parent
246dae949c
commit
a6a1ef05f0
6 files changed
+265
-15
No files matched your search
+77
-9
@@ -21,8 +21,8 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from bench.metrics import aggregate, score_query
|
||||
from bench.query_set import load_queries, scale_probes
|
||||
from bench.metrics import aggregate, newest_gold_first, score_query
|
||||
from bench.query_set import load_queries, scale_probes, source_times
|
||||
from bench.report import git_commit
|
||||
from bench.run import latency
|
||||
from bench.scratch import copy_store
|
||||
@@ -189,6 +189,32 @@ def rank_all(service: Any, ids: dict[str, str], probes: list[Any], top_k: int) -
|
||||
return rankings, distinct, timings
|
||||
|
||||
|
||||
def _groups(
|
||||
answerable: list[Any], scores: list[Any], rankings: list[list[str]], times: dict[str, str]
|
||||
) -> dict[str, Any]:
|
||||
"""recall@10 by modality and by the question type the set recorded, plus who came back first."""
|
||||
by_modality: dict[str, list[float]] = {}
|
||||
by_category: dict[str, list[float]] = {}
|
||||
newest: dict[str, list[bool]] = {}
|
||||
every_newest: list[bool] = []
|
||||
for query, score, ranking in zip(answerable, scores, rankings, strict=True):
|
||||
by_modality.setdefault(query.modality, []).append(score.recall_at_10)
|
||||
by_category.setdefault(query.category or "unlabelled", []).append(score.recall_at_10)
|
||||
flag = newest_gold_first(ranking, set(query.gold), times)
|
||||
if flag is not None:
|
||||
newest.setdefault(query.category or "unlabelled", []).append(flag)
|
||||
every_newest.append(flag)
|
||||
return {
|
||||
"by_modality": {name: sum(rows) / len(rows) for name, rows in sorted(by_modality.items())},
|
||||
"modality_counts": {name: len(rows) for name, rows in sorted(by_modality.items())},
|
||||
"by_category": {name: sum(rows) / len(rows) for name, rows in sorted(by_category.items())},
|
||||
"category_counts": {name: len(rows) for name, rows in sorted(by_category.items())},
|
||||
"newest_gold_first": sum(every_newest) / len(every_newest) if every_newest else None,
|
||||
"newest_gold_first_by_category": {name: sum(rows) / len(rows) for name, rows in sorted(newest.items())},
|
||||
"newest_gold_first_counts": {name: len(rows) for name, rows in sorted(newest.items())},
|
||||
}
|
||||
|
||||
|
||||
def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: int, models: dict[str, Path]) -> dict[str, Any]:
|
||||
"""Score every configuration on the same probes, one warm service per configuration."""
|
||||
from ely_eye.config import Settings
|
||||
@@ -207,6 +233,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
||||
if not answerable:
|
||||
raise SystemExit("the frozen suite carries no answerable query; nothing to score")
|
||||
print(f"{len(answerable)} answerable probes", flush=True)
|
||||
times = source_times()
|
||||
|
||||
vectors = _dense_vectors(home / "data" / "ely_eye.sqlite")
|
||||
if vectors == 0 and any(config["dense"] for config in CONFIGS):
|
||||
@@ -253,9 +280,6 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
||||
]
|
||||
summary = aggregate(scores)
|
||||
timing = latency(timings)
|
||||
by_modality: dict[str, list[float]] = {}
|
||||
for query, score in zip(answerable, scores, strict=True):
|
||||
by_modality.setdefault(query.modality, []).append(score.recall_at_10)
|
||||
arms.append(
|
||||
{
|
||||
"label": config["label"],
|
||||
@@ -269,10 +293,7 @@ def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes:
|
||||
"p50_seconds": timing["p50"],
|
||||
"p95_seconds": timing["p95"],
|
||||
"median_distinct_sources": statistics.median(distinct),
|
||||
"by_modality": {
|
||||
modality: sum(rows) / len(rows) for modality, rows in sorted(by_modality.items())
|
||||
},
|
||||
"modality_counts": {modality: len(rows) for modality, rows in sorted(by_modality.items())},
|
||||
**_groups(answerable, scores, rankings, times),
|
||||
}
|
||||
)
|
||||
print(
|
||||
@@ -326,6 +347,37 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
|
||||
+ " |"
|
||||
)
|
||||
counts = report["arms"][0]["modality_counts"]
|
||||
|
||||
def category_table(title: str, field: str, sizes_field: str) -> list[str]:
|
||||
"""A table over the question types the sets label; unlabelled sets stay out of it."""
|
||||
names = sorted({name for arm in report["arms"] for name in arm[field] if name != "unlabelled"})
|
||||
if not names:
|
||||
return []
|
||||
lines = [
|
||||
f"## {title}",
|
||||
"",
|
||||
"| leg configuration | " + " | ".join(names) + " |",
|
||||
"| --- |" + " --- |" * len(names),
|
||||
]
|
||||
for arm in report["arms"]:
|
||||
values, sizes = arm[field], arm[sizes_field]
|
||||
cells = []
|
||||
for name in names:
|
||||
if name not in values:
|
||||
cells.append("—")
|
||||
elif field == "by_category":
|
||||
cells.append(f"{values[name]:.3f}")
|
||||
else:
|
||||
cells.append(f"{values[name]:.2f} ({sizes[name]})")
|
||||
lines.append(f"| {arm['label']} | " + " | ".join(cells) + " |")
|
||||
labelled = report["arms"][0][sizes_field]
|
||||
lines += [
|
||||
"",
|
||||
"_Queries per type: "
|
||||
+ ", ".join(f"{name} {size}" for name, size in labelled.items() if name != "unlabelled")
|
||||
+ "._",
|
||||
]
|
||||
return lines
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
@@ -375,6 +427,22 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
|
||||
"",
|
||||
f"_Query counts per modality: {', '.join(f'{name} {count}' for name, count in counts.items())}._",
|
||||
"",
|
||||
*category_table("recall@10 by question type", "by_category", "category_counts"),
|
||||
"",
|
||||
*category_table(
|
||||
"the newest of the gold memories came back first",
|
||||
"newest_gold_first_by_category",
|
||||
"newest_gold_first_counts",
|
||||
),
|
||||
"",
|
||||
(
|
||||
"recall@10 counts any gold memory, so a knowledge-update question can score 1.000 "
|
||||
"while the ranking still leads with the value the speaker has since replaced. The "
|
||||
"table above asks the sharper question: when the answer lives in several dated "
|
||||
"memories, is the newest one the first one returned? Only questions with at least two "
|
||||
"dated gold memories and at least one returned are counted."
|
||||
),
|
||||
"",
|
||||
(
|
||||
"The picture leg is not a switch this harness flips per row: the product runs it only "
|
||||
"when the sparse page comes back thin, so these rows measure the product with the leg "
|
||||
|
||||
Reference in new issue
Block a user