ZacharyZhang-NY
9cb46c18d3
The 500-line rule and the one-line-comment rule were enforced only from
memory, so they drifted. bench/filesize_check.py now checks the budget on
every tracked source file, and it passes.
Split, with behaviour proven unchanged by comparing each moved function body
against HEAD byte for byte (53 training bodies, 21 benchmark bodies, 0
differences):
db.py 503 -> 396 store_search.py 105
bench/run.py 609 -> 351 query_set.py 80, report.py 211
benchmarks.py 619 -> 424 benchmark_scoring.py 262
training.py 1630 -> 94 training_{common,encode,metrics,datasets,lora,retrieval}
TrainingService stays as a thin facade for the CLI; the facade's parameter
lists are pinned against the module functions by inspect, which is what
catches a leaked self.
Deleted the eight top-level functions no file called, after fixing the scan
that had reported 201 false positives by skipping the defining file. The
v1/v2/v3 sweep found nothing to delete and says so: every remaining version
tag is an on-disk format marker, a migration step, a fixture name, or a
third-party model name.
Collapsed 39 multi-line comment blocks (86 lines) to one line each, keeping
the measured facts and error strings that justify the code.
269 tests pass; ruff clean; mypy 24 errors in 9 files against a 24-in-6
baseline, net zero; pyright 4, the same four HEAD measures.
109 lines
3.7 KiB
Python
109 lines
3.7 KiB
Python
"""Shared constants, digests and trace writers: the vocabulary every training module speaks."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shutil
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any, TextIO
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from .schemas import TrainingAdapterKind
|
|
|
|
SYSTEM_PROMPT = (
|
|
"You are Ely-Eye. Answer from supplied evidence, return compact JSON, "
|
|
"and cite atom ids exactly."
|
|
)
|
|
|
|
TRAINING_PROOF_VERSION = 2
|
|
|
|
TRAINING_METHOD = "BF16 LoRA"
|
|
|
|
TRAINING_PRECISION = "bf16"
|
|
|
|
TRAINING_FRAMEWORK = "unsloth_patched_transformers_peft"
|
|
|
|
GRADIENT_CHECKPOINTING_MODE = "unsloth_patched_torch_non_reentrant"
|
|
|
|
AUTOCAST_DTYPE = "bf16"
|
|
|
|
class TrainingSample(BaseModel):
|
|
id: str
|
|
task: str
|
|
inputs: list[dict[str, Any]]
|
|
memory: dict[str, Any] = Field(default_factory=dict)
|
|
output: dict[str, Any]
|
|
|
|
class _DigestWriter:
|
|
def __init__(self, digest: hashlib._Hash) -> None:
|
|
self.digest = digest
|
|
|
|
def write(self, data: bytes) -> int:
|
|
self.digest.update(data)
|
|
return len(data)
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
shutil.copyfileobj(handle, _DigestWriter(digest))
|
|
return digest.hexdigest()
|
|
|
|
def sha256_tree(root: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
for path in sorted(root.rglob("*")):
|
|
if path.is_file() and path.name != "adapter_manifest.json":
|
|
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
|
with path.open("rb") as handle:
|
|
shutil.copyfileobj(handle, _DigestWriter(digest))
|
|
return digest.hexdigest()
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
def slugify(value: str) -> str:
|
|
slug = re.sub(r"[^a-zA-Z0-9\u4e00-\u9fff]+", "-", value).strip("-").lower()
|
|
return slug[:80] or "section"
|
|
|
|
def adapter_id(kind: TrainingAdapterKind, dataset_path: Path, model_id: str) -> str:
|
|
raw = f"{kind.value}:{model_id}:{dataset_path.resolve()}:{datetime.now(UTC).isoformat()}"
|
|
return f"{kind.value}_{sha256_bytes(raw.encode('utf-8'))[:12]}"
|
|
|
|
def relative_artifacts(root: Path) -> dict[str, str]:
|
|
artifacts: dict[str, str] = {}
|
|
for path in sorted(root.rglob("*")):
|
|
if path.is_file() and path.name != "adapter_manifest.json":
|
|
artifacts[path.stem] = path.relative_to(root).as_posix()
|
|
return artifacts
|
|
|
|
def write_jsonl(path: Path, samples: list[TrainingSample]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
|
for sample in samples:
|
|
handle.write(sample.model_dump_json() + "\n")
|
|
|
|
def write_trace(handle: TextIO, payload: dict[str, Any]) -> None:
|
|
handle.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
|
|
handle.flush()
|
|
|
|
def write_training_summary(path: Path, payload: dict[str, Any]) -> None:
|
|
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
def split_markdown_sections(text: str) -> list[dict[str, str]]:
|
|
matches = list(re.finditer(r"^(#{2,3})\s+(.+?)\s*$", text, flags=re.MULTILINE))
|
|
sections: list[dict[str, str]] = []
|
|
for index, match in enumerate(matches):
|
|
start = match.end()
|
|
end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
|
|
sections.append({"title": match.group(2).strip(), "body": text[start:end].strip()})
|
|
return sections
|
|
|
|
def section_by_title(text: str, title_fragment: str) -> str:
|
|
for section in split_markdown_sections(text):
|
|
if title_fragment in section["title"]:
|
|
return section["body"]
|
|
return ""
|