Let every harness work on a copy of the frozen store
copy_store lived in dense_scale, so the two harnesses that also needed it reached back into a measurement module for a plumbing helper. It now sits in systems beside the product path it protects, and bench.run passes a scratch directory to the live check: the product path migrates whatever database it opens, and the frozen baseline is cited evidence.
This commit is contained in:
1 parent
b8f83f422a
commit
91fbff34ee
5 files changed
+39
-22
No files matched your search
+15
-2
@@ -73,6 +73,7 @@ def live_check(
|
||||
fast_rankings: dict[str, list[str]],
|
||||
top_k: int = 10,
|
||||
live_queries: int = LIVE_QUERIES,
|
||||
scratch: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Score the unmodified RetrievalService on its own, as its holder would ask it.
|
||||
|
||||
@@ -85,7 +86,11 @@ def live_check(
|
||||
if not chosen:
|
||||
raise SystemExit("live check needs answerable queries")
|
||||
rankings, distinct, timings = live_product_rankings(
|
||||
DB_HOME, [query.text for query in chosen], top_k, as_holders=[query.holder for query in chosen]
|
||||
DB_HOME,
|
||||
[query.text for query in chosen],
|
||||
top_k,
|
||||
as_holders=[query.holder for query in chosen],
|
||||
scratch=scratch,
|
||||
)
|
||||
overlaps = [
|
||||
len(set(fast_rankings[query.query_id]) & set(ranking)) / top_k
|
||||
@@ -229,6 +234,12 @@ def main() -> int:
|
||||
parser.add_argument("--limit", type=int, default=None, help="score at most this many queries per modality")
|
||||
parser.add_argument("--no-live", action="store_true", help="skip the unmodified RetrievalService path")
|
||||
parser.add_argument("--live-queries", type=int, default=LIVE_QUERIES)
|
||||
parser.add_argument(
|
||||
"--live-scratch",
|
||||
type=Path,
|
||||
default=ROOT / "runs" / "live",
|
||||
help="copy the store here before the product path opens it, so the frozen baseline stays frozen",
|
||||
)
|
||||
parser.add_argument("--no-scale", action="store_true", help="skip the latency/scale curve")
|
||||
parser.add_argument(
|
||||
"--remeasure-latency",
|
||||
@@ -325,7 +336,9 @@ def main() -> int:
|
||||
# A crash in the live or scale phase should not cost every finished arm.
|
||||
(DB_HOME / "partial_report.json").write_text(json.dumps(report, indent=1, sort_keys=True), encoding="utf-8")
|
||||
if not args.no_live:
|
||||
report["live_product"] = live_check(queries, fast_rankings, live_queries=args.live_queries)
|
||||
report["live_product"] = live_check(
|
||||
queries, fast_rankings, live_queries=args.live_queries, scratch=args.live_scratch
|
||||
)
|
||||
if not args.no_scale:
|
||||
report["scale"] = scale_curve(queries, atoms, [arm for arm in arms_requested if arm in ("bm25_only", "dense_only")])
|
||||
|
||||
|
||||
Reference in new issue
Block a user