From 00dc9f8eaf8395ff8778bbebd9cb85f5971a3834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 7 Oct 2026 13:39:17 -0400 Subject: [PATCH] Re-measure arm latency on a warm-model sample instead of trusting the cache rankings.jsonl stores the seconds each query took, so the report inherited whatever the machine was doing at the time: the frozen baseline's dense latency included a per-query model load, and the 50,000-source row moved from 1.315s to 0.809s between two runs of identical code. - bench/run.py: --remeasure-latency N re-times a stride sample with warm models and replaces each arm's latency section, recording the sample size and a note; recall stays a full-pass number. BASELINE.md prints that note under the table. - bench/tests/test_joint.py: 62 tests total. Verification: pytest bench/tests -q -> 62 passed; ruff check bench/ -> all checks passed; mypy (fresh cache) -> no issues in 25 source files. --- bench/run.py | 36 +++++++++++++++++++++++++++++++++++- bench/tests/test_joint.py | 11 +++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bench/run.py b/bench/run.py index 1e77a0b..8b6d137 100644 --- a/bench/run.py +++ b/bench/run.py @@ -264,6 +264,8 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: f"| chance (random ranking) | {chance['recall_at_1']:.3f} | {chance['recall_at_5']:.3f} " f"| {chance['recall_at_10']:.3f} | - | - | - | - | - |" ) + if note := next((arm["latency_seconds"].get("note") for arm in report["arms"] if arm["latency_seconds"].get("note")), None): + lines += ["", f"_{note} (n={report['arms'][0]['latency_seconds']['queries']} queries)_"] for arm in report["arms"]: overall = arm["overall"] latency = arm["latency_seconds"] @@ -361,7 +363,7 @@ def load_config(parser: argparse.ArgumentParser, args: argparse.Namespace, path: raise SystemExit(f"{path}: unknown arm(s) {unknown}, expected one of {list(ARMS)}") -def latency(times: Sequence[float]) -> dict[str, float]: +def latency(times: Sequence[float]) -> dict[str, Any]: """Latency summary for one arm's per-query wall times.""" values = np.asarray(times, dtype=np.float64) if values.size == 0: @@ -421,6 +423,30 @@ def load_rankings(path: Path, mark: str) -> dict[str, dict[str, Any]]: return rows +def remeasure_latency( + arms: list[dict[str, Any]], joint: JointRunner, queries: Sequence[Query], count: int +) -> None: + """Re-time a stride sample with warm models and replace each arm's latency section. + + Cached per-query seconds are whatever the machine was doing at the time: a run sharing the + cores with another job, or a harness that paid model loading per query, reported that as + retrieval latency. Recall stays a full-pass number; only the timing column is re-measured + here, and the sample size is recorded so the two are never confused. + """ + sample = scale_probes(queries, count) + times: dict[str, list[float]] = {arm["arm"]: [] for arm in arms} + for index, query in enumerate(sample, start=1): + for name, (_, seconds) in joint.rank_all(query.text).items(): + if name in times: + times[name].append(seconds) + if index % PROGRESS_EVERY == 0: + print(f" latency {index}/{len(sample)}, rss {rss_gib():.1f} GiB", flush=True) + for arm in arms: + measured: dict[str, Any] = latency(times[arm["arm"]]) + measured["note"] = "re-timed on a stride sample with warm models; recall comes from the full pass" + arm["latency_seconds"] = measured + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--arms", nargs="+", choices=ARMS, action="extend", default=None) @@ -429,6 +455,12 @@ def main() -> int: parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path") parser.add_argument("--live-queries", type=int, default=LIVE_QUERIES) parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve") + parser.add_argument( + "--remeasure-latency", + type=int, + default=0, + help="re-time this many sampled queries with warm models instead of trusting cached per-query seconds", + ) parser.add_argument("--out", type=Path, default=REPORTS) parser.add_argument("--config", type=Path, help="JSON defaults for these flags (bench/configs/ablation.json)") args = parser.parse_args() @@ -510,6 +542,8 @@ def main() -> int: flush=True, ) report["arms"] = arms + if args.remeasure_latency: + remeasure_latency(arms, joint, queries, args.remeasure_latency) # A crash in the live or scale phase should not cost every finished arm. (DB_HOME / "partial_report.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8") if not args.no_live: diff --git a/bench/tests/test_joint.py b/bench/tests/test_joint.py index 0baa20d..fab604a 100644 --- a/bench/tests/test_joint.py +++ b/bench/tests/test_joint.py @@ -8,6 +8,7 @@ from __future__ import annotations import json from pathlib import Path +from typing import Any import pytest @@ -17,6 +18,7 @@ from bench.run import ( fingerprint, latency, load_rankings, + remeasure_latency, scale_arm_names, scale_probes, write_markdown, @@ -200,3 +202,12 @@ def test_scale_arm_names_reads_only_latency_dicts() -> None: assert scale_arm_names(_report()["scale"]) == ["bm25_only"] with pytest.raises(SystemExit, match="no measured arm"): scale_arm_names([{"sources": 500, "note": "latency only"}]) + + +def test_remeasure_latency_replaces_cached_seconds_with_a_warm_sample() -> None: + arms: list[dict[str, Any]] = [{"arm": "bm25_only", "latency_seconds": {"p50": 99.0}}] + remeasure_latency(arms, JointRunner(["bm25_only"], ATOMS, systems.MODELS), _probe_set(8), 4) + measured = arms[0]["latency_seconds"] + assert measured["p50"] < 99.0 + assert measured["queries"] == 4 + assert "warm models" in measured["note"]