Keep every file inside the rules the repo already wrote down

The 500-line rule and the one-line-comment rule were enforced only from
memory, so they drifted. bench/filesize_check.py now checks the budget on
every tracked source file, and it passes.

Split, with behaviour proven unchanged by comparing each moved function body
against HEAD byte for byte (53 training bodies, 21 benchmark bodies, 0
differences):

  db.py        503 -> 396   store_search.py     105
  bench/run.py 609 -> 351   query_set.py         80, report.py 211
  benchmarks.py 619 -> 424  benchmark_scoring.py 262
  training.py 1630 ->  94   training_{common,encode,metrics,datasets,lora,retrieval}

TrainingService stays as a thin facade for the CLI; the facade's parameter
lists are pinned against the module functions by inspect, which is what
catches a leaked self.

Deleted the eight top-level functions no file called, after fixing the scan
that had reported 201 false positives by skipping the defining file. The
v1/v2/v3 sweep found nothing to delete and says so: every remaining version
tag is an on-disk format marker, a migration step, a fixture name, or a
third-party model name.

Collapsed 39 multi-line comment blocks (86 lines) to one line each, keeping
the measured facts and error strings that justify the code.

269 tests pass; ruff clean; mypy 24 errors in 9 files against a 24-in-6
baseline, net zero; pyright 4, the same four HEAD measures.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-08 08:20:21 -04:00
1 parent 56b94cd58e
commit 9cb46c18d3
47 files changed
+2678 -2272

No files matched your search

+6 -14
View File
@@ -35,9 +35,7 @@ REPO = Path(__file__).resolve().parent.parent
RUNS = REPO / "bench" / "runs" / "baseline"
SHARDS = RUNS / "shards"
TABLES = ("sources", "atoms", "atom_fts", "cartridges", "cache_events", "evidence_edges")
# Virtual-address ceiling per worker. ONNX Runtime and torch reserve far more address space than
# they touch, so a 3 GiB ceiling killed OCR on large images ("Unable to allocate 582. MiB for an
# array with shape (5632, 9024, 3)") even though resident memory stayed near 1.5 GiB.
# ONNX Runtime and torch reserve far more address space than they touch, so a 3 GiB ceiling killed OCR on large images ("Unable to allocate 582. MiB for an array with shape (5632, 9024, 3)") while resident memory stayed near 1.5 GiB.
WORKER_MEMORY_BYTES = 6 * 1024**3
MAX_POOL_ATTEMPTS = 5
DELETE_STATEMENTS = (
@@ -125,8 +123,7 @@ def _pin_inference_threads() -> None:
options = ort.SessionOptions()
options.intra_op_num_threads = 1
options.inter_op_num_threads = 1
# The CPU arena keeps freed blocks mapped, so a worker that OCRs thousands of pages grows its
# resident set until it hits its ceiling on an 11 MiB array. Hand the memory back instead.
# The CPU arena keeps freed blocks mapped, so a worker OCRing thousands of pages grows until it hits its ceiling on an 11 MiB array.
options.enable_cpu_mem_arena = False
kwargs["sess_options"] = options
return original(model_path, *args, **kwargs)
@@ -197,8 +194,7 @@ def ingest_shard(job: tuple[int, list[Path]]) -> dict:
with ledger.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(outcome) + "\n")
if ingested % 25 == 0:
# ONNX Runtime keeps the address space it has touched, and one shard runs thousands of
# files, so the arena has to be handed back or the worker hits its ceiling on an 11 MiB array.
# ONNX Runtime keeps the address space it touched, so one shard of thousands of files has to hand the arena back.
gc.collect()
elapsed = time.monotonic() - started
print(f"shard-{index:02d} {ingested}/{len(paths)} files in {elapsed / 60:.1f} min", flush=True)
@@ -310,9 +306,7 @@ def merge() -> dict:
copied_objects += 1
connection.commit()
# A file processed in two attempts lands in two shards. The atoms table
# dedupes on atom_id; fts5 has no key and kept every copy (288,266 rows for
# 165,973 atoms), which would inflate the product's text search statistics.
# A file processed twice lands in two shards: atoms dedupe on atom_id but fts5 has no key and kept every copy (288,266 rows for 165,973 atoms).
connection.execute("DELETE FROM atom_fts")
connection.execute(
"INSERT INTO atom_fts (atom_id, source, text) SELECT atom_id, source, text FROM atoms"
@@ -335,8 +329,7 @@ def merge() -> dict:
raise SystemExit(f"{main_db}: {len(unmapped)} sources are not named by the corpus index, first {unmapped[:3]}")
product_paths = {path for path, _ in rows}
product_ids = {source_id for _, source_id in rows}
# The product names a source by content hash, so two byte-identical files
# collapse into one source row (500 duplicate LongMemEval sessions).
# The product names a source by content hash, so byte-identical files collapse into one source row (500 duplicate LongMemEval sessions).
collapsed_input = sorted(path for path in done - product_paths if safe_source_id(Path(path), sha256_bytes(Path(path).read_bytes())) in product_ids)
unaccounted = sorted(done - product_paths - set(collapsed_input))
if unaccounted:
@@ -363,8 +356,7 @@ def merge() -> dict:
)
counts["uncovered_sources"] = len(total - covered)
if counts["files_not_ingested"] == 0 and covered != total:
# Two honest reasons: byte-identical inputs collapse into one content-hashed
# source, and files the product cannot ingest at all. Anything else is a bug.
# Two honest reasons: byte-identical inputs collapse into one content-hashed source, and files the product cannot ingest at all.
explained = counts["duplicate_content_files"] + counts["files_failed"]
if counts["uncovered_sources"] > explained:
raise SystemExit(