Re-measure arm latency on a warm-model sample instead of trusting the cache
rankings.jsonl stores the seconds each query took, so the report inherited whatever the machine was doing at the time: the frozen baseline's dense latency included a per-query model load, and the 50,000-source row moved from 1.315s to 0.809s between two runs of identical code. - bench/run.py: --remeasure-latency N re-times a stride sample with warm models and replaces each arm's latency section, recording the sample size and a note; recall stays a full-pass number. BASELINE.md prints that note under the table. - bench/tests/test_joint.py: 62 tests total. Verification: pytest bench/tests -q -> 62 passed; ruff check bench/ -> all checks passed; mypy (fresh cache) -> no issues in 25 source files.
This commit is contained in:
1 parent
dbe1d2cc82
commit
00dc9f8eaf
2 files changed
+46
-1
No files matched your search
+35
-1
@@ -264,6 +264,8 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
|
|||||||
f"| chance (random ranking) | {chance['recall_at_1']:.3f} | {chance['recall_at_5']:.3f} "
|
f"| chance (random ranking) | {chance['recall_at_1']:.3f} | {chance['recall_at_5']:.3f} "
|
||||||
f"| {chance['recall_at_10']:.3f} | - | - | - | - | - |"
|
f"| {chance['recall_at_10']:.3f} | - | - | - | - | - |"
|
||||||
)
|
)
|
||||||
|
if note := next((arm["latency_seconds"].get("note") for arm in report["arms"] if arm["latency_seconds"].get("note")), None):
|
||||||
|
lines += ["", f"_{note} (n={report['arms'][0]['latency_seconds']['queries']} queries)_"]
|
||||||
for arm in report["arms"]:
|
for arm in report["arms"]:
|
||||||
overall = arm["overall"]
|
overall = arm["overall"]
|
||||||
latency = arm["latency_seconds"]
|
latency = arm["latency_seconds"]
|
||||||
@@ -361,7 +363,7 @@ def load_config(parser: argparse.ArgumentParser, args: argparse.Namespace, path:
|
|||||||
raise SystemExit(f"{path}: unknown arm(s) {unknown}, expected one of {list(ARMS)}")
|
raise SystemExit(f"{path}: unknown arm(s) {unknown}, expected one of {list(ARMS)}")
|
||||||
|
|
||||||
|
|
||||||
def latency(times: Sequence[float]) -> dict[str, float]:
|
def latency(times: Sequence[float]) -> dict[str, Any]:
|
||||||
"""Latency summary for one arm's per-query wall times."""
|
"""Latency summary for one arm's per-query wall times."""
|
||||||
values = np.asarray(times, dtype=np.float64)
|
values = np.asarray(times, dtype=np.float64)
|
||||||
if values.size == 0:
|
if values.size == 0:
|
||||||
@@ -421,6 +423,30 @@ def load_rankings(path: Path, mark: str) -> dict[str, dict[str, Any]]:
|
|||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def remeasure_latency(
|
||||||
|
arms: list[dict[str, Any]], joint: JointRunner, queries: Sequence[Query], count: int
|
||||||
|
) -> None:
|
||||||
|
"""Re-time a stride sample with warm models and replace each arm's latency section.
|
||||||
|
|
||||||
|
Cached per-query seconds are whatever the machine was doing at the time: a run sharing the
|
||||||
|
cores with another job, or a harness that paid model loading per query, reported that as
|
||||||
|
retrieval latency. Recall stays a full-pass number; only the timing column is re-measured
|
||||||
|
here, and the sample size is recorded so the two are never confused.
|
||||||
|
"""
|
||||||
|
sample = scale_probes(queries, count)
|
||||||
|
times: dict[str, list[float]] = {arm["arm"]: [] for arm in arms}
|
||||||
|
for index, query in enumerate(sample, start=1):
|
||||||
|
for name, (_, seconds) in joint.rank_all(query.text).items():
|
||||||
|
if name in times:
|
||||||
|
times[name].append(seconds)
|
||||||
|
if index % PROGRESS_EVERY == 0:
|
||||||
|
print(f" latency {index}/{len(sample)}, rss {rss_gib():.1f} GiB", flush=True)
|
||||||
|
for arm in arms:
|
||||||
|
measured: dict[str, Any] = latency(times[arm["arm"]])
|
||||||
|
measured["note"] = "re-timed on a stride sample with warm models; recall comes from the full pass"
|
||||||
|
arm["latency_seconds"] = measured
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
parser.add_argument("--arms", nargs="+", choices=ARMS, action="extend", default=None)
|
parser.add_argument("--arms", nargs="+", choices=ARMS, action="extend", default=None)
|
||||||
@@ -429,6 +455,12 @@ def main() -> int:
|
|||||||
parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path")
|
parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path")
|
||||||
parser.add_argument("--live-queries", type=int, default=LIVE_QUERIES)
|
parser.add_argument("--live-queries", type=int, default=LIVE_QUERIES)
|
||||||
parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve")
|
parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve")
|
||||||
|
parser.add_argument(
|
||||||
|
"--remeasure-latency",
|
||||||
|
type=int,
|
||||||
|
default=0,
|
||||||
|
help="re-time this many sampled queries with warm models instead of trusting cached per-query seconds",
|
||||||
|
)
|
||||||
parser.add_argument("--out", type=Path, default=REPORTS)
|
parser.add_argument("--out", type=Path, default=REPORTS)
|
||||||
parser.add_argument("--config", type=Path, help="JSON defaults for these flags (bench/configs/ablation.json)")
|
parser.add_argument("--config", type=Path, help="JSON defaults for these flags (bench/configs/ablation.json)")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -510,6 +542,8 @@ def main() -> int:
|
|||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
report["arms"] = arms
|
report["arms"] = arms
|
||||||
|
if args.remeasure_latency:
|
||||||
|
remeasure_latency(arms, joint, queries, args.remeasure_latency)
|
||||||
# A crash in the live or scale phase should not cost every finished arm.
|
# A crash in the live or scale phase should not cost every finished arm.
|
||||||
(DB_HOME / "partial_report.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8")
|
(DB_HOME / "partial_report.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8")
|
||||||
if not args.no_live:
|
if not args.no_live:
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ from bench.run import (
|
|||||||
fingerprint,
|
fingerprint,
|
||||||
latency,
|
latency,
|
||||||
load_rankings,
|
load_rankings,
|
||||||
|
remeasure_latency,
|
||||||
scale_arm_names,
|
scale_arm_names,
|
||||||
scale_probes,
|
scale_probes,
|
||||||
write_markdown,
|
write_markdown,
|
||||||
@@ -200,3 +202,12 @@ def test_scale_arm_names_reads_only_latency_dicts() -> None:
|
|||||||
assert scale_arm_names(_report()["scale"]) == ["bm25_only"]
|
assert scale_arm_names(_report()["scale"]) == ["bm25_only"]
|
||||||
with pytest.raises(SystemExit, match="no measured arm"):
|
with pytest.raises(SystemExit, match="no measured arm"):
|
||||||
scale_arm_names([{"sources": 500, "note": "latency only"}])
|
scale_arm_names([{"sources": 500, "note": "latency only"}])
|
||||||
|
|
||||||
|
|
||||||
|
def test_remeasure_latency_replaces_cached_seconds_with_a_warm_sample() -> None:
|
||||||
|
arms: list[dict[str, Any]] = [{"arm": "bm25_only", "latency_seconds": {"p50": 99.0}}]
|
||||||
|
remeasure_latency(arms, JointRunner(["bm25_only"], ATOMS, systems.MODELS), _probe_set(8), 4)
|
||||||
|
measured = arms[0]["latency_seconds"]
|
||||||
|
assert measured["p50"] < 99.0
|
||||||
|
assert measured["queries"] == 4
|
||||||
|
assert "warm models" in measured["note"]
|
||||||
Reference in new issue
Block a user