Re-measure arm latency on a warm-model sample instead of trusting the cache
rankings.jsonl stores the seconds each query took, so the report inherited whatever the machine was doing at the time: the frozen baseline's dense latency included a per-query model load, and the 50,000-source row moved from 1.315s to 0.809s between two runs of identical code. - bench/run.py: --remeasure-latency N re-times a stride sample with warm models and replaces each arm's latency section, recording the sample size and a note; recall stays a full-pass number. BASELINE.md prints that note under the table. - bench/tests/test_joint.py: 62 tests total. Verification: pytest bench/tests -q -> 62 passed; ruff check bench/ -> all checks passed; mypy (fresh cache) -> no issues in 25 source files.
This commit is contained in:
1 parent
dbe1d2cc82
commit
00dc9f8eaf
2 files changed
+46
-1
No files matched your search
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -17,6 +18,7 @@ from bench.run import (
|
||||
fingerprint,
|
||||
latency,
|
||||
load_rankings,
|
||||
remeasure_latency,
|
||||
scale_arm_names,
|
||||
scale_probes,
|
||||
write_markdown,
|
||||
@@ -200,3 +202,12 @@ def test_scale_arm_names_reads_only_latency_dicts() -> None:
|
||||
assert scale_arm_names(_report()["scale"]) == ["bm25_only"]
|
||||
with pytest.raises(SystemExit, match="no measured arm"):
|
||||
scale_arm_names([{"sources": 500, "note": "latency only"}])
|
||||
|
||||
|
||||
def test_remeasure_latency_replaces_cached_seconds_with_a_warm_sample() -> None:
|
||||
arms: list[dict[str, Any]] = [{"arm": "bm25_only", "latency_seconds": {"p50": 99.0}}]
|
||||
remeasure_latency(arms, JointRunner(["bm25_only"], ATOMS, systems.MODELS), _probe_set(8), 4)
|
||||
measured = arms[0]["latency_seconds"]
|
||||
assert measured["p50"] < 99.0
|
||||
assert measured["queries"] == 4
|
||||
assert "warm models" in measured["note"]
|
||||
Reference in new issue
Block a user