Add baseline harness: ingest, metrics, ablation arms, external counterpart, audit
- bench/ingest_baseline.py drives the unmodified product IngestionService over all 50,789 corpus files in sharded SQLite DBs, then merges them. Thread and memory ceilings are set explicitly: ONNX Runtime defaulted to every core (95 threads/worker, load 490) and its CPU arena kept freed pages mapped, which OOM-killed workers. - bench/metrics.py: recall@k, MRR, nDCG@10, Wilson lower bound, chance baseline. No model, no product import, so the scorer cannot grade itself. - bench/systems.py + bench/run.py: the five ablation arms over the merged DB, latency percentiles, scale curve, and a top-10 equivalence check against the live RetrievalService. - bench/counterpart.py + bench/run_external.py: in-repo Mem0-style flat store and Zep-style temporal store as the external reference point. - bench/audit.py: enforces single-attempt scoring, frozen n, a chance row beside every score, and producer != verifier. It already flags the product's retry-until-target loop at backend/src/ely_eye/evals.py:2351. - backend/pyproject.toml: rapidocr-onnxruntime>=1.4.4 was unsatisfiable (PyPI maxes at 1.2.3); the successor is the renamed rapidocr package.
This commit is contained in:
1 parent
77219c930b
commit
fc0698c242
27 files changed
+2589
-24
No files matched your search
+359
@@ -0,0 +1,359 @@
|
||||
"""Run the retrieval ablation over the frozen corpus and write the baseline report.
|
||||
|
||||
Five arms, one database: sparse only, dense only, graph only, the product's weighted
|
||||
fusion, and fusion plus a cross-encoder reranker. Every number in the report is either
|
||||
measured or explicitly marked as chance.
|
||||
|
||||
Run: python -m bench.run [--arms bm25_only] [--limit 200] [--no-live] [--out bench/reports]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from bench.metrics import aggregate, chance_recall_at_k, score_query
|
||||
from bench.sets_common import MODALITIES, SETS, read_jsonl, sha256_file
|
||||
from bench.systems import (
|
||||
ARMS,
|
||||
DENSE_MODEL,
|
||||
MODELS,
|
||||
RERANK_MODEL,
|
||||
ArmRunner,
|
||||
Atom,
|
||||
arm_needs,
|
||||
live_product_rankings,
|
||||
load_atoms,
|
||||
)
|
||||
|
||||
ROOT = SETS.parent
|
||||
REPORTS = ROOT / "reports"
|
||||
DB_HOME = ROOT / "runs" / "baseline"
|
||||
LIVE_QUERIES = 120
|
||||
SCALE_SIZES = (500, 2_000, 10_000, 50_000)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Query:
|
||||
query_id: str
|
||||
text: str
|
||||
modality: str
|
||||
gold: frozenset[str]
|
||||
holder: str | None
|
||||
answerable: bool
|
||||
as_of: str | None
|
||||
|
||||
|
||||
def load_queries(modalities: Sequence[str] = MODALITIES) -> list[Query]:
|
||||
"""Every query the suite froze, in file order."""
|
||||
queries: list[Query] = []
|
||||
for modality in modalities:
|
||||
path = SETS / f"{modality}.jsonl"
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"{path}: missing set, run python -m bench.build_sets first")
|
||||
for record in read_jsonl(path):
|
||||
gold = frozenset(hit["source_id"] for hit in record["gold"])
|
||||
answerable = not bool(record["unanswerable"])
|
||||
if answerable != bool(gold):
|
||||
raise SystemExit(
|
||||
f"{path}: {record['query_id']} carries {len(gold)} gold hits but "
|
||||
f"unanswerable={record['unanswerable']}"
|
||||
)
|
||||
queries.append(
|
||||
Query(
|
||||
query_id=record["query_id"],
|
||||
text=record["question"],
|
||||
modality=modality,
|
||||
gold=gold,
|
||||
holder=record["scope"]["holder"],
|
||||
answerable=answerable,
|
||||
as_of=record.get("as_of"),
|
||||
)
|
||||
)
|
||||
if not queries:
|
||||
raise SystemExit("no queries loaded")
|
||||
return queries
|
||||
|
||||
|
||||
def holder_index(modalities: Sequence[str] = MODALITIES) -> dict[str, str]:
|
||||
"""source_id -> holder, from the corpus side of the suite."""
|
||||
holders: dict[str, str] = {}
|
||||
for modality in modalities:
|
||||
path = SETS / "corpus" / f"{modality}.jsonl"
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"{path}: missing corpus index, run python -m bench.build_sets first")
|
||||
for record in read_jsonl(path):
|
||||
holders[record["source_id"]] = record["holder"]
|
||||
return holders
|
||||
|
||||
|
||||
def chance_row(queries: Sequence[Query], corpus_sources: int) -> dict[str, float]:
|
||||
"""What a random ranking would score on the same queries."""
|
||||
answerable = [query for query in queries if query.answerable]
|
||||
if not answerable:
|
||||
raise SystemExit("chance needs answerable queries")
|
||||
return {
|
||||
"corpus_sources": corpus_sources,
|
||||
"recall_at_1": sum(chance_recall_at_k(len(query.gold), corpus_sources, 1) for query in answerable) / len(answerable),
|
||||
"recall_at_5": sum(chance_recall_at_k(len(query.gold), corpus_sources, 5) for query in answerable) / len(answerable),
|
||||
"recall_at_10": sum(chance_recall_at_k(len(query.gold), corpus_sources, 10) for query in answerable) / len(answerable),
|
||||
}
|
||||
|
||||
|
||||
def scale_curve(queries: Sequence[Query], atoms: Sequence[Atom], arms: Sequence[str]) -> list[dict[str, Any]]:
|
||||
"""Latency of each arm as the corpus grows, measured on nested source prefixes."""
|
||||
sources = sorted({atom.source_id for atom in atoms})
|
||||
rows: list[dict[str, Any]] = []
|
||||
sample = [query for query in queries if query.answerable][:40]
|
||||
for size in SCALE_SIZES:
|
||||
keep = set(sources[:size])
|
||||
if not keep:
|
||||
raise SystemExit(f"scale size {size} kept no sources")
|
||||
subset = [atom for atom in atoms if atom.source_id in keep]
|
||||
probe = [query for query in sample if query.gold & keep]
|
||||
if not probe:
|
||||
continue
|
||||
row: dict[str, Any] = {"sources": len(keep), "atoms": len(subset), "queries": len(probe)}
|
||||
for arm in arms:
|
||||
if not arm_needs(arm, MODELS):
|
||||
continue
|
||||
runner = ArmRunner(arm, subset, MODELS)
|
||||
for query in probe:
|
||||
runner.rank(query.text)
|
||||
row[arm] = {"p50": runner.percentile(0.50), "p95": runner.percentile(0.95)}
|
||||
rows.append(row)
|
||||
print(f"scale {len(keep):6d} sources: " + " ".join(f"{arm}={row[arm]['p50']:.3f}s" for arm in arms if arm in row), flush=True)
|
||||
return rows
|
||||
|
||||
|
||||
def live_check(queries: Sequence[Query], fast_rankings: dict[str, list[str]], top_k: int = 10) -> dict[str, Any]:
|
||||
"""Compare the unmodified RetrievalService with the fast BM25 path."""
|
||||
probe = [query for query in queries if query.answerable]
|
||||
stride = max(1, len(probe) // LIVE_QUERIES)
|
||||
chosen = probe[::stride][:LIVE_QUERIES]
|
||||
if not chosen:
|
||||
raise SystemExit("live check needs answerable queries")
|
||||
rankings, timings = live_product_rankings(DB_HOME, [query.text for query in chosen], top_k)
|
||||
overlaps = [
|
||||
len(set(fast_rankings[query.query_id]) & set(ranking)) / top_k
|
||||
for query, ranking in zip(chosen, rankings, strict=True)
|
||||
if fast_rankings.get(query.query_id) is not None
|
||||
]
|
||||
if not overlaps:
|
||||
raise SystemExit("live check found no fast rankings to compare against")
|
||||
return {
|
||||
"queries": len(chosen),
|
||||
"p50": sorted(timings)[len(timings) // 2],
|
||||
"mean": sum(timings) / len(timings),
|
||||
"max": max(timings),
|
||||
"top10_overlap_with_bm25_only": sum(overlaps) / len(overlaps),
|
||||
"identical_top10": sum(1 for overlap in overlaps if overlap == 1.0) / len(overlaps),
|
||||
}
|
||||
|
||||
|
||||
def git_commit() -> str:
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True, check=True, timeout=30
|
||||
)
|
||||
commit = result.stdout.strip()
|
||||
if len(commit) != 40:
|
||||
raise SystemExit(f"git rev-parse HEAD returned {commit!r}, expected a 40 character sha")
|
||||
return commit
|
||||
|
||||
|
||||
def meta(corpus_sources: int, queries: Sequence[Query], limit: int | None = None) -> dict[str, Any]:
|
||||
manifest_path = SETS / "manifest.json"
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise SystemExit(f"{manifest_path}: cannot read the frozen set manifest: {error}") from error
|
||||
return {
|
||||
"generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"git_commit": git_commit(),
|
||||
"python": platform.python_version(),
|
||||
"machine": {"system": platform.system(), "release": platform.release(), "cpus": os.cpu_count()},
|
||||
"corpus_manifest_sha256": manifest.get("corpus_manifest_sha256"),
|
||||
"sets": {name: sha256_file(SETS / f"{name}.jsonl") for name in MODALITIES},
|
||||
"queries": len(queries),
|
||||
"answerable": sum(1 for query in queries if query.answerable),
|
||||
"limit_per_modality": limit,
|
||||
"corpus_sources": corpus_sources,
|
||||
"models": {"dense": DENSE_MODEL, "rerank": RERANK_MODEL},
|
||||
}
|
||||
|
||||
|
||||
def write_markdown(report: dict[str, Any], path: Path) -> None:
|
||||
"""Human-readable twin of the JSON report."""
|
||||
lines = [
|
||||
"# BASELINE: what the product retrieves today",
|
||||
"",
|
||||
f"- generated {report['meta']['generated_at']} at commit {report['meta']['git_commit'][:12]}",
|
||||
(
|
||||
f"- corpus {report['meta']['corpus_sources']} sources, {report['meta']['queries']} queries "
|
||||
f"({report['meta']['answerable']} answerable)"
|
||||
),
|
||||
f"- models: dense `{report['meta']['models']['dense']}`, rerank `{report['meta']['models']['rerank']}`",
|
||||
"",
|
||||
"| arm | recall@1 | recall@5 | recall@10 | nDCG@10 | MRR | hit@10 (95% lower) | p50 s | p95 s |",
|
||||
"|---|---|---|---|---|---|---|---|---|",
|
||||
]
|
||||
chance = report["chance"]
|
||||
lines.append(
|
||||
f"| chance (random ranking) | {chance['recall_at_1']:.3f} | {chance['recall_at_5']:.3f} "
|
||||
f"| {chance['recall_at_10']:.3f} | - | - | - | - | - |"
|
||||
)
|
||||
for arm in report["arms"]:
|
||||
overall = arm["overall"]
|
||||
latency = arm["latency_seconds"]
|
||||
lines.append(
|
||||
f"| {arm['arm']} | {overall['recall_at_1']:.3f} | {overall['recall_at_5']:.3f} "
|
||||
f"| {overall['recall_at_10']:.3f} | {overall['ndcg_at_10']:.3f} | {overall['mrr']:.3f} "
|
||||
f"| {overall['hit_at_10']:.3f} ({overall['hit_at_10_wilson_lower']:.3f}) "
|
||||
f"| {latency['p50']:.3f} | {latency['p95']:.3f} |"
|
||||
)
|
||||
lines += ["", "## recall@10 by modality", "", "| arm | " + " | ".join(MODALITIES) + " |", "|---|" + "---|" * len(MODALITIES)]
|
||||
for arm in report["arms"]:
|
||||
cells = [f"{arm['by_modality'][modality]['recall_at_10']:.3f}" for modality in MODALITIES]
|
||||
lines.append(f"| {arm['arm']} | " + " | ".join(cells) + " |")
|
||||
lines += ["", "## unanswerable queries (44.8% of the suite)", ""]
|
||||
for arm in report["arms"]:
|
||||
overall = arm["overall"]
|
||||
lines.append(
|
||||
f"- {arm['arm']}: abstains on {overall['abstention_rate_unanswerable']:.0%} of unanswerable queries, "
|
||||
f"and puts a wrong-holder source on top for {overall['wrong_holder_confidence_unanswerable']:.0%} of them"
|
||||
)
|
||||
if "live_product" in report:
|
||||
live = report["live_product"]
|
||||
lines += [
|
||||
"",
|
||||
"## the unmodified product path",
|
||||
"",
|
||||
(
|
||||
f"- RetrievalService.search: p50 {live['p50']:.2f}s, mean {live['mean']:.2f}s, max {live['max']:.2f}s "
|
||||
f"over {live['queries']} queries"
|
||||
),
|
||||
(
|
||||
f"- top-10 overlap with the fast BM25 path: {live['top10_overlap_with_bm25_only']:.1%}, "
|
||||
f"identical order {live['identical_top10']:.1%}"
|
||||
),
|
||||
]
|
||||
if report.get("scale"):
|
||||
lines += ["", "## latency vs corpus size", "", "| sources | atoms | " + " | ".join(a for a in report["scale"][0] if a not in ("sources", "atoms", "queries")) + " |", "|---" * (3 + len(report["scale"][0]) - 3) + "|"]
|
||||
for row in report["scale"]:
|
||||
cells = [f"{row[arm]['p50']:.3f}/{row[arm]['p95']:.3f}" for arm in row if arm not in ("sources", "atoms", "queries")]
|
||||
lines.append(f"| {row['sources']} | {row['atoms']} | " + " | ".join(cells) + " |")
|
||||
lines += [
|
||||
"",
|
||||
"## reading",
|
||||
"",
|
||||
"- recall counts a source as found only when it is the source a third party marked correct.",
|
||||
"- abstention is the only honest number for unanswerable queries: a retriever that always",
|
||||
" returns ten results never says it does not know.",
|
||||
"- chance is the floor any of these arms must beat; a number near chance is not a capability.",
|
||||
]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def load_config(parser: argparse.ArgumentParser, args: argparse.Namespace, path: Path) -> None:
|
||||
"""Fill in whatever the command line left at its default; an explicit flag still wins."""
|
||||
try:
|
||||
config = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise SystemExit(f"{path}: cannot read the run config ({error})") from error
|
||||
if not isinstance(config, dict):
|
||||
raise SystemExit(f"{path}: expected a JSON object, got {type(config).__name__}")
|
||||
for key, value in config.items():
|
||||
if key not in vars(args):
|
||||
raise SystemExit(f"{path}: unknown key {key!r}, expected one of {sorted(vars(args))}")
|
||||
if getattr(args, key) == parser.get_default(key):
|
||||
setattr(args, key, value)
|
||||
unknown = [arm for arm in args.arms if arm not in ARMS]
|
||||
if unknown:
|
||||
raise SystemExit(f"{path}: unknown arm(s) {unknown}, expected one of {list(ARMS)}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--arms", nargs="+", choices=ARMS, default=list(ARMS))
|
||||
parser.add_argument("--modality", action="append", choices=MODALITIES, default=None)
|
||||
parser.add_argument("--limit", type=int, default=None, help="score at most this many queries per modality")
|
||||
parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path")
|
||||
parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve")
|
||||
parser.add_argument("--out", type=Path, default=REPORTS)
|
||||
parser.add_argument("--config", type=Path, help="JSON defaults for these flags (bench/configs/ablation.json)")
|
||||
args = parser.parse_args()
|
||||
if args.config:
|
||||
load_config(parser, args, args.config)
|
||||
|
||||
modalities = args.modality or list(MODALITIES)
|
||||
queries = load_queries(modalities)
|
||||
if args.limit:
|
||||
capped: list[Query] = []
|
||||
for modality in modalities:
|
||||
picked = [query for query in queries if query.modality == modality][: args.limit]
|
||||
capped.extend(picked)
|
||||
queries = capped
|
||||
atoms = load_atoms(DB_HOME / "data" / "ely_eye.sqlite")
|
||||
holders = holder_index(modalities)
|
||||
corpus_sources = len({atom.source_id for atom in atoms})
|
||||
print(f"{len(queries)} queries, {len(atoms)} atoms, {corpus_sources} sources", flush=True)
|
||||
|
||||
report: dict[str, Any] = {
|
||||
"meta": meta(corpus_sources, queries, args.limit),
|
||||
"chance": chance_row(queries, corpus_sources),
|
||||
}
|
||||
arms: list[dict[str, Any]] = []
|
||||
fast_rankings: dict[str, list[str]] = {}
|
||||
for arm in args.arms:
|
||||
print(f"== {arm}", flush=True)
|
||||
runner = ArmRunner(arm, atoms, MODELS)
|
||||
scores = []
|
||||
for query in queries:
|
||||
ranked = runner.rank(query.text)
|
||||
scores.append(score_query(query.query_id, ranked, set(query.gold), holder=query.holder, holder_of=holders))
|
||||
if arm == "bm25_only":
|
||||
fast_rankings[query.query_id] = ranked
|
||||
summary: dict[str, Any] = {"arm": arm, "overall": aggregate(scores)}
|
||||
summary["by_modality"] = {
|
||||
modality: aggregate([s for s, q in zip(scores, queries, strict=True) if q.modality == modality])
|
||||
for modality in sorted({q.modality for q in queries})
|
||||
}
|
||||
summary["latency_seconds"] = {
|
||||
"queries": len(runner.timings),
|
||||
"mean": sum(runner.timings) / len(runner.timings),
|
||||
"p50": runner.percentile(0.50),
|
||||
"p95": runner.percentile(0.95),
|
||||
"max": max(runner.timings),
|
||||
}
|
||||
arms.append(summary)
|
||||
print(
|
||||
f" recall@10 {summary['overall']['recall_at_10']:.3f} "
|
||||
f"hit@10 {summary['overall']['hit_at_10']:.3f} "
|
||||
f"(95% lower {summary['overall']['hit_at_10_wilson_lower']:.3f}) "
|
||||
f"abstain {summary['overall']['abstention_rate_unanswerable']:.0%} "
|
||||
f"p95 {summary['latency_seconds']['p95']:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
report["arms"] = arms
|
||||
if not args.no_live:
|
||||
report["live_product"] = live_check(queries, fast_rankings)
|
||||
if not args.no_scale:
|
||||
report["scale"] = scale_curve(queries, atoms, [arm for arm in args.arms if arm in ("bm25_only", "dense_only")])
|
||||
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
(args.out / "baseline.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8")
|
||||
write_markdown(report, args.out / "BASELINE.md")
|
||||
print(f"== wrote {args.out / 'baseline.json'} and {args.out / 'BASELINE.md'}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user