Measure what each retrieval leg is worth before choosing a default
The product fuses a sparse page, a dense vector leg, and a picture leg that only speaks when the words come back thin. Two of the three are off by default, and until now no committed artifact said why. bench/leg_check.py asks the frozen suite's 300 answerable queries as their own holder, once per configuration, and prints recall, first-hit quality, latency and per-modality recall in one table. The picture index and the CLIP snapshot are 48 minutes of encoding, so they are carried in rather than rebuilt per row; the harness refuses to run a leg whose snapshot is missing instead of quietly scoring nothing. The numbers that came back: - words only, 10 atoms: recall@10 0.832, MRR 0.683, p50 64ms - words only, the configured 24: 0.848, 0.688, 69ms - words only, page 30: 0.855, 0.690, 67ms - dense leg on: 0.845 / 0.858, MRR down, p50 469ms / 411ms - picture leg available: 0.815 / 0.825, MRR 0.553 / 0.551, p50 148ms / 164ms So both defaults stay off, and the acceptance bar (recall@10 >= 0.90) is not met by the product path yet: 0.858 is the best measured row, the in-repo counterpart at holder scope is 0.866. The loss concentrates in picture golds, and the current CLIP leg pays more in dilution than it returns there. CHECKLIST p2-1 and p0-3.7 now quote the committed reports (DENSE.md, QUANT.md, SCALE.md, TERMS.md) instead of scratch scripts.
This commit is contained in:
ZacharyZhang-NY committed
1 parent
91fbff34ee
commit
dff5977ab6
3 files changed
+389
-7
No files matched your search
@@ -0,0 +1,315 @@
|
||||
"""What each retrieval leg is actually worth, on the frozen suite, asked as its holder.
|
||||
|
||||
The product fuses three legs with reciprocal rank fusion: a sparse page, a dense vector leg, and a
|
||||
picture leg that only speaks up when the words come back thin. Each one costs latency and a model
|
||||
snapshot, and the suite is the only honest referee. This harness turns each leg on and off over the
|
||||
same probes and prints recall, first-hit quality, latency, and per-modality recall in one table, so
|
||||
a default like `dense_embeddings_enabled=False` is a measurement rather than a preference.
|
||||
|
||||
Run: python -m bench.leg_check [--probes 300]
|
||||
Writes: bench/reports/LEGS.md, bench/reports/leg_check.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from bench.metrics import aggregate, score_query
|
||||
from bench.query_set import load_queries, scale_probes
|
||||
from bench.report import git_commit
|
||||
from bench.run import latency
|
||||
from bench.systems import copy_store, machine_facts
|
||||
|
||||
DEFAULT_HOME = Path("bench/runs/dense_scale")
|
||||
DEFAULT_SCRATCH = Path("bench/runs/leg_check")
|
||||
DEFAULT_OBJECTS = Path("bench/runs/baseline/objects")
|
||||
DEFAULT_CLIP_INDEX = Path("bench/runs/leg_check_index")
|
||||
DEFAULT_CLIP_MODEL = Path("bench/models/chinese-clip-vit-base-patch16")
|
||||
DEFAULT_EMBEDDING_MODEL = Path("bench/models/paraphrase-multilingual-MiniLM-L12-v2")
|
||||
|
||||
# The page size the product asks for by default. measure() checks it against the product's own
|
||||
# setting, so a config change here cannot quietly turn the "as shipped" row into a hypothetical.
|
||||
SHIPPED_TOP_K = 24
|
||||
|
||||
# (label, atoms asked, dense leg, picture leg). The picture leg routes itself: it answers only when
|
||||
# the sparse page comes back thin, which is the product's rule, not this harness's.
|
||||
CONFIGS: list[dict[str, Any]] = [
|
||||
{"label": "words only", "top_k": 10, "dense": False, "visual": False},
|
||||
{"label": "words only (the configured 24)", "top_k": SHIPPED_TOP_K, "dense": False, "visual": False},
|
||||
{"label": "words only, wider page", "top_k": 30, "dense": False, "visual": False},
|
||||
{"label": "+ dense leg", "top_k": 10, "dense": True, "visual": False},
|
||||
{"label": "+ dense leg, wider page", "top_k": 30, "dense": True, "visual": False},
|
||||
{"label": "+ picture leg", "top_k": 10, "dense": False, "visual": True},
|
||||
{"label": "+ picture leg, wider page", "top_k": 30, "dense": False, "visual": True},
|
||||
]
|
||||
|
||||
|
||||
def prepare(
|
||||
home: Path,
|
||||
scratch: Path,
|
||||
objects: Path,
|
||||
clip_index: Path,
|
||||
*,
|
||||
dense: bool,
|
||||
visual: bool,
|
||||
clip_model: Path,
|
||||
embedding_model: Path,
|
||||
) -> Path:
|
||||
"""Build a scratch store that can run every leg: vectors, the picture index, and the bytes.
|
||||
|
||||
The picture index and the CLIP snapshot are 48 minutes of encoding, so they are carried in from
|
||||
wherever they were built rather than rebuilt per row.
|
||||
"""
|
||||
copy_home, _ = copy_store(home, scratch)
|
||||
if visual:
|
||||
for name in ("clip_image_index.npy", "clip_image_index_atoms.json"):
|
||||
source = clip_index / name
|
||||
if not source.exists():
|
||||
raise SystemExit(
|
||||
f"{source}: the picture index has not been built; run `ely-eye index-visual` "
|
||||
"with ELY_EYE_CLIP_MODEL_DIR set, or point --clip-index at its cache"
|
||||
)
|
||||
(copy_home / "cache").mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, copy_home / "cache" / name)
|
||||
if not (clip_model / "onnx" / "model_fp16.onnx").exists():
|
||||
raise SystemExit(f"{clip_model / 'onnx' / 'model_fp16.onnx'}: the CLIP snapshot is missing")
|
||||
if not objects.is_dir():
|
||||
raise SystemExit(f"{objects}: the picture leg needs the object store to check image bytes")
|
||||
link = copy_home / "objects"
|
||||
if not link.exists() and not link.is_symlink():
|
||||
link.symlink_to(objects.resolve())
|
||||
if dense and not (embedding_model / "config.json").exists():
|
||||
raise SystemExit(f"{embedding_model}: the embedding snapshot is missing; the dense leg cannot run")
|
||||
|
||||
return copy_home
|
||||
|
||||
|
||||
def _schema(home: Path) -> int:
|
||||
import sqlite3
|
||||
|
||||
with sqlite3.connect(home / "data" / "ely_eye.sqlite") as connection:
|
||||
return int(connection.execute("PRAGMA user_version").fetchone()[0])
|
||||
|
||||
|
||||
def rank_all(service: Any, ids: dict[str, str], probes: list[Any], top_k: int) -> tuple[list[list[str]], list[int], list[float]]:
|
||||
"""Ask the product as each probe's own holder and collapse atoms to sources by first hit."""
|
||||
from ely_eye.schemas import SearchFilters
|
||||
|
||||
rankings: list[list[str]] = []
|
||||
distinct: list[int] = []
|
||||
timings: list[float] = []
|
||||
for index, query in enumerate(probes):
|
||||
began = time.perf_counter()
|
||||
hits = service.search(query.text, top_k=top_k, filters=SearchFilters(as_holder=query.holder)).hits
|
||||
timings.append(time.perf_counter() - began)
|
||||
sources: list[str] = []
|
||||
for hit in hits:
|
||||
source = ids[hit.atom.source_id]
|
||||
if source not in sources:
|
||||
sources.append(source)
|
||||
distinct.append(len(sources))
|
||||
rankings.append(sources)
|
||||
if (index + 1) % 100 == 0:
|
||||
print(f" {index + 1}/{len(probes)}", flush=True)
|
||||
return rankings, distinct, timings
|
||||
|
||||
|
||||
def measure(home: Path, scratch: Path, objects: Path, clip_index: Path, probes: int, models: dict[str, Path]) -> dict[str, Any]:
|
||||
"""Score every configuration on the same probes, one warm service per configuration."""
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.retrieval import RetrievalService
|
||||
|
||||
shipped = Settings().retrieval_top_k
|
||||
if shipped != SHIPPED_TOP_K:
|
||||
raise SystemExit(
|
||||
f"the product asks for {shipped} atoms per search but this harness pins "
|
||||
f"SHIPPED_TOP_K={SHIPPED_TOP_K}; update the rows before trusting the 'as shipped' label"
|
||||
)
|
||||
|
||||
chosen = scale_probes(load_queries(), probes)
|
||||
answerable = [query for query in chosen if query.gold]
|
||||
if not answerable:
|
||||
raise SystemExit("the frozen suite carries no answerable query; nothing to score")
|
||||
print(f"{len(answerable)} answerable probes", flush=True)
|
||||
|
||||
arms: list[dict[str, Any]] = []
|
||||
schema = 0
|
||||
for config in CONFIGS:
|
||||
print(f" {config['label']}: dense={config['dense']} visual={config['visual']}", flush=True)
|
||||
copy_home = prepare(
|
||||
home,
|
||||
scratch,
|
||||
objects,
|
||||
clip_index,
|
||||
dense=config["dense"],
|
||||
visual=config["visual"],
|
||||
clip_model=models["clip"],
|
||||
embedding_model=models["embedding"],
|
||||
)
|
||||
schema = _schema(copy_home)
|
||||
settings = Settings(
|
||||
home=copy_home,
|
||||
dense_embeddings_enabled=config["dense"],
|
||||
embedding_model_dir=models["embedding"] if config["dense"] else None,
|
||||
clip_model_dir=models["clip"] if config["visual"] else None,
|
||||
)
|
||||
database = Database(settings)
|
||||
service = RetrievalService(settings=settings, database=database)
|
||||
ids = _source_ids(database)
|
||||
rankings, distinct, timings = rank_all(service, ids, answerable, int(config["top_k"]))
|
||||
scores = [
|
||||
score_query(query.query_id, ranking, set(query.gold), query.holder)
|
||||
for query, ranking in zip(answerable, rankings, strict=True)
|
||||
]
|
||||
summary = aggregate(scores)
|
||||
timing = latency(timings)
|
||||
by_modality: dict[str, list[float]] = {}
|
||||
for query, score in zip(answerable, scores, strict=True):
|
||||
by_modality.setdefault(query.modality, []).append(score.recall_at_10)
|
||||
arms.append(
|
||||
{
|
||||
"label": config["label"],
|
||||
"top_k": config["top_k"],
|
||||
"dense": config["dense"],
|
||||
"visual": config["visual"],
|
||||
"recall_at_10": summary["recall_at_10"],
|
||||
"mrr": summary["mrr"],
|
||||
"hit_at_10": summary["hit_at_10"],
|
||||
"wilson_lower": summary["hit_at_10_wilson_lower"],
|
||||
"p50_seconds": timing["p50"],
|
||||
"p95_seconds": timing["p95"],
|
||||
"median_distinct_sources": statistics.median(distinct),
|
||||
"by_modality": {
|
||||
modality: sum(rows) / len(rows) for modality, rows in sorted(by_modality.items())
|
||||
},
|
||||
"modality_counts": {modality: len(rows) for modality, rows in sorted(by_modality.items())},
|
||||
}
|
||||
)
|
||||
print(
|
||||
f" recall@10 {summary['recall_at_10']:.3f}, mrr {summary['mrr']:.3f}, "
|
||||
f"p50 {timing['p50'] * 1000:.1f}ms",
|
||||
flush=True,
|
||||
)
|
||||
return {
|
||||
"commit": git_commit(),
|
||||
"schema_version": schema,
|
||||
"store": str(scratch / "data" / "ely_eye.sqlite"),
|
||||
"probes": len(answerable),
|
||||
"arms": arms,
|
||||
"machine": machine_facts(),
|
||||
}
|
||||
|
||||
|
||||
def _source_ids(database: Any) -> dict[str, str]:
|
||||
"""Store source id to benchmark source id, the same mapping every other arm uses."""
|
||||
from bench.systems import _corpus_ids
|
||||
|
||||
return _corpus_ids(database)
|
||||
|
||||
|
||||
def write_markdown(report: dict[str, Any], path: Path) -> None:
|
||||
machine = report["machine"]
|
||||
header = [
|
||||
"| leg configuration | atoms asked | recall@10 | MRR | hit@10 | 95% lower | p50 ms | p95 ms | distinct sources |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
rows = []
|
||||
for arm in report["arms"]:
|
||||
rows.append(
|
||||
f"| {arm['label']} | {arm['top_k']} | {arm['recall_at_10']:.3f} | {arm['mrr']:.3f} | "
|
||||
f"{arm['hit_at_10']:.3f} | {arm['wilson_lower']:.3f} | {arm['p50_seconds'] * 1000:.1f} | "
|
||||
f"{arm['p95_seconds'] * 1000:.1f} | {arm['median_distinct_sources']} |"
|
||||
)
|
||||
modalities = sorted({modality for arm in report["arms"] for modality in arm["by_modality"]})
|
||||
modality_rows = [
|
||||
"| leg configuration | " + " | ".join(modalities) + " |",
|
||||
"| --- |" + " --- |" * len(modalities),
|
||||
]
|
||||
for arm in report["arms"]:
|
||||
modality_rows.append(
|
||||
f"| {arm['label']} | "
|
||||
+ " | ".join(f"{arm['by_modality'].get(modality, float('nan')):.3f}" for modality in modalities)
|
||||
+ " |"
|
||||
)
|
||||
counts = report["arms"][0]["modality_counts"]
|
||||
path.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"# LEGS: what each retrieval leg is worth",
|
||||
"",
|
||||
f"- commit: `{report['commit']}`, schema v{report['schema_version']}",
|
||||
f"- store: {report['store']} (a copy of {DEFAULT_HOME}; the frozen baseline is never opened)",
|
||||
f"- {report['probes']} answerable queries, each asked as the holder the frozen set assigns it",
|
||||
(
|
||||
f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, "
|
||||
f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"
|
||||
),
|
||||
"",
|
||||
"Recall is scored on the first ten **sources** after atoms are collapsed to their source",
|
||||
"by first hit, which is how every other arm in this repo is scored.",
|
||||
"",
|
||||
*header,
|
||||
*rows,
|
||||
"",
|
||||
"## recall@10 by modality",
|
||||
"",
|
||||
*modality_rows,
|
||||
"",
|
||||
f"_Query counts per modality: {', '.join(f'{name} {count}' for name, count in counts.items())}._",
|
||||
"",
|
||||
(
|
||||
"The picture leg is not a switch this harness flips per row: the product runs it only "
|
||||
"when the sparse page comes back thin, so these rows measure the product with the leg "
|
||||
"available, not a forced always-on fusion."
|
||||
),
|
||||
"",
|
||||
(
|
||||
"_Latency is wall time per search on the machine named above with the thread budget "
|
||||
"the benchmark pins; a run on a loaded machine is not comparable._"
|
||||
),
|
||||
"",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--home", type=Path, default=DEFAULT_HOME)
|
||||
parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH)
|
||||
parser.add_argument("--objects", type=Path, default=DEFAULT_OBJECTS)
|
||||
parser.add_argument("--clip-index", type=Path, default=DEFAULT_CLIP_INDEX)
|
||||
parser.add_argument("--clip-model", type=Path, default=DEFAULT_CLIP_MODEL)
|
||||
parser.add_argument("--embedding-model", type=Path, default=DEFAULT_EMBEDDING_MODEL)
|
||||
parser.add_argument("--probes", type=int, default=300)
|
||||
parser.add_argument("--out", type=Path, default=Path("bench/reports/LEGS.md"))
|
||||
parser.add_argument("--json-out", type=Path, default=Path("bench/reports/leg_check.json"))
|
||||
args = parser.parse_args()
|
||||
|
||||
report = measure(
|
||||
args.home,
|
||||
args.scratch,
|
||||
args.objects,
|
||||
args.clip_index,
|
||||
args.probes,
|
||||
{"clip": args.clip_model, "embedding": args.embedding_model},
|
||||
)
|
||||
args.json_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.json_out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
write_markdown(report, args.out)
|
||||
print(f"wrote {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.version_info < (3, 11):
|
||||
raise SystemExit("bench needs Python 3.11+")
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""The leg ablation must stay an ablation: every leg on and off, and a report that says so."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from bench import leg_check
|
||||
|
||||
|
||||
def test_every_leg_appears_switched_both_ways() -> None:
|
||||
dense = {config["dense"] for config in leg_check.CONFIGS}
|
||||
visual = {config["visual"] for config in leg_check.CONFIGS}
|
||||
assert dense == {True, False}, "the dense leg is only an ablation if both states are scored"
|
||||
assert visual == {True, False}, "the picture leg is only an ablation if both states are scored"
|
||||
pages = {int(config["top_k"]) for config in leg_check.CONFIGS}
|
||||
assert leg_check.SHIPPED_TOP_K in pages, "the page size the product ships must be one of the rows"
|
||||
|
||||
|
||||
def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
|
||||
arm = {
|
||||
"label": "words only",
|
||||
"top_k": 10,
|
||||
"dense": False,
|
||||
"visual": False,
|
||||
"recall_at_10": 0.832,
|
||||
"mrr": 0.683,
|
||||
"hit_at_10": 0.833,
|
||||
"wilson_lower": 0.787,
|
||||
"p50_seconds": 0.0637,
|
||||
"p95_seconds": 0.13,
|
||||
"median_distinct_sources": 4,
|
||||
"by_modality": {"code": 0.953, "pdf": 0.849},
|
||||
"modality_counts": {"code": 128, "pdf": 86},
|
||||
}
|
||||
report: dict[str, Any] = {
|
||||
"commit": "abcdef123456",
|
||||
"schema_version": 8,
|
||||
"store": "bench/runs/leg_check/data/ely_eye.sqlite",
|
||||
"probes": 300,
|
||||
"arms": [arm],
|
||||
"machine": {
|
||||
"system": "Linux",
|
||||
"release": "6.18",
|
||||
"cpus": 32,
|
||||
"threads_per_query": 4,
|
||||
"load_average_1_5_15": [1.0, 2.0, 3.0],
|
||||
},
|
||||
}
|
||||
out = tmp_path / "LEGS.md"
|
||||
leg_check.write_markdown(report, out)
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert "words only" in text
|
||||
assert "0.832" in text and "63.7" in text, "recall and latency must sit in the same row"
|
||||
assert "code" in text and "0.953" in text, "per-modality recall is where the picture legs show up"
|
||||
assert "abcdef123456" in text and "load_average" not in text
|
||||
Reference in new issue
Block a user