ZacharyZhang-NY
9cb46c18d3
The 500-line rule and the one-line-comment rule were enforced only from
memory, so they drifted. bench/filesize_check.py now checks the budget on
every tracked source file, and it passes.
Split, with behaviour proven unchanged by comparing each moved function body
against HEAD byte for byte (53 training bodies, 21 benchmark bodies, 0
differences):
db.py 503 -> 396 store_search.py 105
bench/run.py 609 -> 351 query_set.py 80, report.py 211
benchmarks.py 619 -> 424 benchmark_scoring.py 262
training.py 1630 -> 94 training_{common,encode,metrics,datasets,lora,retrieval}
TrainingService stays as a thin facade for the CLI; the facade's parameter
lists are pinned against the module functions by inspect, which is what
catches a leaked self.
Deleted the eight top-level functions no file called, after fixing the scan
that had reported 201 false positives by skipping the defining file. The
v1/v2/v3 sweep found nothing to delete and says so: every remaining version
tag is an on-disk format marker, a migration step, a fixture name, or a
third-party model name.
Collapsed 39 multi-line comment blocks (86 lines) to one line each, keeping
the measured facts and error strings that justify the code.
269 tests pass; ruff clean; mypy 24 errors in 9 files against a 24-in-6
baseline, net zero; pyright 4, the same four HEAD measures.
84 lines
2.8 KiB
Python
84 lines
2.8 KiB
Python
"""The frozen query set: what is asked, who asks it, and which probes to time."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
from dataclasses import dataclass
|
|
|
|
from bench.sets_common import MODALITIES, SETS, read_jsonl
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Query:
|
|
query_id: str
|
|
text: str
|
|
modality: str
|
|
gold: frozenset[str]
|
|
holder: str | None
|
|
answerable: bool
|
|
as_of: str | None
|
|
|
|
|
|
|
|
def load_queries(modalities: Sequence[str] = MODALITIES) -> list[Query]:
|
|
"""Every query the suite froze, in file order."""
|
|
queries: list[Query] = []
|
|
for modality in modalities:
|
|
path = SETS / f"{modality}.jsonl"
|
|
if not path.is_file():
|
|
raise SystemExit(f"{path}: missing set, run python -m bench.build_sets first")
|
|
for record in read_jsonl(path):
|
|
gold = frozenset(hit["source_id"] for hit in record["gold"])
|
|
answerable = not bool(record["unanswerable"])
|
|
if answerable != bool(gold):
|
|
raise SystemExit(
|
|
f"{path}: {record['query_id']} carries {len(gold)} gold hits but "
|
|
f"unanswerable={record['unanswerable']}"
|
|
)
|
|
queries.append(
|
|
Query(
|
|
query_id=record["query_id"],
|
|
text=record["question"],
|
|
modality=modality,
|
|
gold=gold,
|
|
holder=record["scope"]["holder"],
|
|
answerable=answerable,
|
|
as_of=record.get("as_of"),
|
|
)
|
|
)
|
|
if not queries:
|
|
raise SystemExit("no queries loaded")
|
|
return queries
|
|
|
|
|
|
|
|
def holder_index(modalities: Sequence[str] = MODALITIES) -> dict[str, str]:
|
|
"""source_id -> holder, from the corpus side of the suite."""
|
|
holders: dict[str, str] = {}
|
|
for modality in modalities:
|
|
path = SETS / "corpus" / f"{modality}.jsonl"
|
|
if not path.is_file():
|
|
raise SystemExit(f"{path}: missing corpus index, run python -m bench.build_sets first")
|
|
for record in read_jsonl(path):
|
|
holders[record["source_id"]] = record["holder"]
|
|
return holders
|
|
|
|
|
|
|
|
SCALE_PROBES = 40
|
|
|
|
|
|
def scale_probes(queries: Sequence[Query], count: int = SCALE_PROBES) -> list[Query]:
|
|
"""Queries to time at every corpus size.
|
|
|
|
Latency does not need gold hits, and a truncated corpus keeps almost none of them,
|
|
so selecting probes by gold would silently drop the small sizes entirely.
|
|
"""
|
|
if count <= 0:
|
|
raise SystemExit("the scale curve needs at least one probe query")
|
|
answerable = [query for query in queries if query.answerable]
|
|
if not answerable:
|
|
raise SystemExit("the scale curve needs answerable queries")
|
|
stride = max(1, len(answerable) // count)
|
|
return answerable[::stride][:count]
|