ZacharyZhang-NY/ely-eye
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make the proof suite say what it actually ran

Browse Source 
The suite had fifteen checks that could not lose. The visual HashHop arena
painted a cyan line and a white ring on the answer slot, then asked the model
to find the answer by looking; the collector re-rolled puzzles until the
running rate crossed its target; the cache check passed if any layer had ever
recorded an entry; the repository check passed on a call graph built in memory
without touching the store; the research-module check asserted non-empty lists
of SHA256-derived placeholder vectors and so could not fail.

Each check now answers for one claim and reports a verdict the reader can act
on: passed, failed, deferred, or removed. A deferred row names what is missing
and the command that would settle it; a removed row says why the old claim was
withdrawn. The suite verdict is failed > deferred > passed, so a claim that
never ran holds the whole report back from saying "proven".

- evals.py (2427 lines) becomes the proofs/ package; largest module 445 lines.
- proofs/__main__.py deleted: it duplicated PRDProofSuite.latest() with a
  different glob. proofs/table.py renders; latest-proof-suite is the one face
  and exits 1 on a failed verdict.
- The arena paints nothing. The puzzle id is printed on its tile instead,
  because removing the ink without it removes the question.
- HashHop runs a fixed 8 puzzles per depth instead of re-rolling to target.
- The contradiction corpus moves from code to contradiction_scenarios.json;
  the check stays deferred while the corpus is the one this repo authored.
- The cache check compiles a nonce question over a real atom twice and reads
  the miss to hit movement; an empty store fails instead of agreeing by luck.
- The repository check requires rows in the store; the research-module check is
  removed.

Two product bugs surfaced on the way:
- text_index.fts_query joined Latin terms with a space, which FTS5 reads as
  AND, so a question longer than its answer retrieved nothing. Terms are ORed
  and bm25 rank orders the candidates.
- profiles.windows_path_to_wsl prefixed /mnt/ onto an already-POSIX path with
  an empty drive, producing /mnt//home/... and failing every launcher probe.

Verification: 168 passed (pytest backend/tests bench/tests); ruff clean on the
proofs package, cli.py, text_index.py, profiles.py, and both test files; mypy
clean on 16 files with a fresh cache; pyright reports 0 errors on proofs/ and
the test file. Ruff debt elsewhere is unchanged: 19 errors in seven files this
commit does not touch, measured against the same files at HEAD.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 18:39:46 -04:00
1 parent dfc6cd4c28
commit 4ab36cd5eb
27 files changed
+4284 -2453
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/src/ely_eye/proofs/training_trace.py
+358
View File
@@ -0,0 +1,358 @@
"""Check that an adapter's training artifacts tell a consistent story.
"""
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Any
from ..schemas import (
AdapterManifest,
TrainingAdapterKind,
)
from ..training import (
AUTOCAST_DTYPE,
GRADIENT_CHECKPOINTING_MODE,
TRAINING_FRAMEWORK,
TRAINING_METHOD,
TRAINING_PRECISION,
TRAINING_PROOF_VERSION,
safetensor_weight_stats,
sha256_file,
)
def count_safetensor_keys(path: Path) -> int:
return int(safetensor_weight_stats(path)["tensor_count"])
def peft_config_loads(path: Path) -> bool:
try:
from peft import PeftConfig
PeftConfig.from_pretrained(str(path))
return True
except Exception: # noqa: BLE001 # a probe records its failure instead of crashing the proof run
return False
def validate_training_contract(manifest: AdapterManifest) -> dict[str, Any]:
target_modules_ok = training_target_modules_ok(manifest.kind, manifest.target_modules)
report = {
"proof_version_ok": manifest.proof_version >= TRAINING_PROOF_VERSION,
"method_ok": manifest.training_method == TRAINING_METHOD,
"precision_ok": manifest.precision == TRAINING_PRECISION and manifest.bf16,
"framework_ok": manifest.framework == TRAINING_FRAMEWORK,
"optimizer_ok": manifest.optimizer == "paged_adamw_8bit",
"optimizer_family_ok": manifest.optimizer_family == "8-bit AdamW / paged optimizer",
"gradient_checkpointing_ok": (
manifest.gradient_checkpointing
and manifest.gradient_checkpointing_mode == GRADIENT_CHECKPOINTING_MODE
),
"autocast_dtype_ok": manifest.autocast_dtype == AUTOCAST_DTYPE,
"unsloth_version_ok": bool(manifest.unsloth_version),
"triton_version_ok": bool(manifest.triton_version),
"xformers_version_ok": bool(manifest.xformers_version),
"unsloth_model_type_ok": bool(manifest.unsloth_model_type),
"max_length_ok": manifest.max_length >= 8192,
"target_modules_ok": target_modules_ok,
}
report["valid"] = all(report.values())
return report
def training_target_modules_ok(kind: TrainingAdapterKind, target_modules: list[str]) -> bool:
targets = set(target_modules)
core = {"q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"}
if kind == TrainingAdapterKind.hme_core_lora:
return core.issubset(targets)
if kind == TrainingAdapterKind.hme_vision_lora:
return bool(targets) and any("visual" in target or "vision" in target for target in targets)
if kind == TrainingAdapterKind.hme_ttt_vl:
has_visual = any("visual" in target or "vision" in target for target in targets)
has_mlp = bool({"gate_proj", "up_proj", "down_proj"} & targets)
return has_visual and has_mlp
if kind == TrainingAdapterKind.hme_visual_mtp:
return any("mtp" in target or "draft" in target or target == "lm_head" for target in targets)
if kind in {TrainingAdapterKind.hme_router, TrainingAdapterKind.hme_retrieval}:
return core.issubset(targets)
return False
def training_payload_contract_ok(payload: dict[str, Any], manifest: AdapterManifest) -> bool:
return (
int(payload.get("proof_version") or 0) >= TRAINING_PROOF_VERSION
and payload.get("training_method") == TRAINING_METHOD
and payload.get("precision") == TRAINING_PRECISION
and payload.get("framework") == TRAINING_FRAMEWORK
and payload.get("optimizer_family") == manifest.optimizer_family
and bool(payload.get("gradient_checkpointing"))
and payload.get("gradient_checkpointing_mode") == GRADIENT_CHECKPOINTING_MODE
and payload.get("autocast_dtype") == AUTOCAST_DTYPE
and payload.get("unsloth_version") == manifest.unsloth_version
and payload.get("triton_version") == manifest.triton_version
and payload.get("xformers_version") == manifest.xformers_version
and payload.get("unsloth_model_type") == manifest.unsloth_model_type
)
def validate_training_trace(manifest: AdapterManifest, adapter_root: Path) -> dict[str, Any]:
trace_path = adapter_root / "training_trace.jsonl"
summary_path = adapter_root / "training_summary.json"
report: dict[str, Any] = {
"trace_path": str(trace_path),
"summary_path": str(summary_path),
"trace_exists": trace_path.exists(),
"summary_exists": summary_path.exists(),
"trace_sha256_ok": False,
"summary_sha256_ok": False,
"step_records": 0,
"loss_history_records": len(manifest.loss_history),
"loss_history_ok": False,
"summary_matches_manifest": False,
"summary_contract_ok": False,
"run_start_contract_ok": False,
"run_finish_contract_ok": False,
"cuda_training": False,
"bf16_supported": False,
"bf16": manifest.bf16,
"valid": False,
}
if not trace_path.exists() or not summary_path.exists():
return report
try:
trace_rows = [
json.loads(line)
for line in trace_path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
summary = json.loads(summary_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return report
step_rows = [row for row in trace_rows if row.get("event") == "step"]
start_row: dict[str, Any] = next((row for row in trace_rows if row.get("event") == "run_start"), {})
finish_row: dict[str, Any] = next(
(row for row in reversed(trace_rows) if row.get("event") == "run_finish"), {}
)
losses = [row.get("loss") for row in step_rows]
finite_losses = all(isinstance(loss, (int, float)) and math.isfinite(float(loss)) for loss in losses)
final_loss_ok = (
manifest.final_loss is not None
and bool(losses)
and abs(float(losses[-1]) - float(manifest.final_loss)) < 1e-6
)
report.update(
{
"trace_sha256_ok": (
manifest.training_trace_sha256 is not None
and sha256_file(trace_path) == manifest.training_trace_sha256
),
"summary_sha256_ok": (
manifest.training_summary_sha256 is not None
and sha256_file(summary_path) == manifest.training_summary_sha256
),
"step_records": len(step_rows),
"loss_history_ok": (
len(manifest.loss_history) == manifest.max_steps
and finite_losses
and final_loss_ok
),
"summary_matches_manifest": (
summary.get("adapter_kind") == manifest.kind.value
and summary.get("base_model") == manifest.base_model
and summary.get("dataset_sha256") == manifest.dataset_sha256
and int(summary.get("max_steps") or -1) == manifest.max_steps
and int(summary.get("trainable_params") or -1) == manifest.trainable_params
),
"summary_contract_ok": training_payload_contract_ok(summary, manifest),
"run_start_contract_ok": training_payload_contract_ok(start_row, manifest),
"run_finish_contract_ok": training_payload_contract_ok(finish_row, manifest),
}
)
environment = summary.get("environment") if isinstance(summary.get("environment"), dict) else {}
device_name = str(environment.get("device_name") or "")
report["cuda_training"] = bool(environment.get("cuda_available")) and "RTX 4090" in device_name
report["bf16_supported"] = bool(environment.get("bf16_supported"))
report["valid"] = (
report["trace_sha256_ok"]
and report["summary_sha256_ok"]
and report["step_records"] == manifest.max_steps
and report["loss_history_ok"]
and report["summary_matches_manifest"]
and report["summary_contract_ok"]
and report["run_start_contract_ok"]
and report["run_finish_contract_ok"]
and report["cuda_training"]
and report["bf16_supported"]
and manifest.bf16
)
return report
def validate_training_proof(manifest: AdapterManifest, adapter_root: Path) -> dict[str, Any]:
proof_path = adapter_root / "training_proof.json"
report: dict[str, Any] = {
"proof_path": str(proof_path),
"proof_exists": proof_path.exists(),
"proof_sha256_ok": False,
"hash_changed": False,
"optimizer_update_count": manifest.optimizer_update_count,
"optimizer_update_count_ok": False,
"max_optimizer_update_l2": manifest.max_optimizer_update_l2,
"max_optimizer_update_l2_ok": False,
"loss_delta": manifest.loss_delta,
"loss_delta_finite": False,
"updates_have_gradients": False,
"updates_have_weight_delta": False,
"proof_weight_sha256_ok": False,
"proof_weight_stats_ok": False,
"proof_cuda_bf16": False,
"proof_bf16_supported": False,
"proof_contract_ok": False,
"proof_optimizer_ok": False,
"proof_max_length_ok": False,
"proof_target_modules_ok": False,
"proof_unsloth_runtime_ok": False,
"proof_matches_manifest": False,
"valid": False,
}
if not proof_path.exists():
return report
try:
proof = json.loads(proof_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return report
updates = proof.get("optimizer_updates") if isinstance(proof.get("optimizer_updates"), list) else []
update_l2_values = [
float(value)
for item in updates
if isinstance(item, dict) and isinstance(value := item.get("update_l2"), (int, float))
]
gradient_l2_values = [
float(value)
for item in updates
if isinstance(item, dict) and isinstance(value := item.get("gradient_l2"), (int, float))
]
weight_report = safetensor_weight_stats(adapter_root / "adapter_model.safetensors")
proof_weight_stats = proof.get("adapter_weight_stats") if isinstance(proof.get("adapter_weight_stats"), dict) else {}
proof_environment = proof.get("environment") if isinstance(proof.get("environment"), dict) else {}
report.update(
{
"proof_sha256_ok": (
manifest.training_proof_sha256 is not None
and sha256_file(proof_path) == manifest.training_proof_sha256
),
"hash_changed": (
bool(manifest.initial_trainable_sha256)
and bool(manifest.final_trainable_sha256)
and manifest.initial_trainable_sha256 != manifest.final_trainable_sha256
),
"optimizer_update_count_ok": (
manifest.optimizer_update_count > 0
and len(updates) == manifest.optimizer_update_count
),
"max_optimizer_update_l2_ok": (
manifest.max_optimizer_update_l2 is not None
and math.isfinite(float(manifest.max_optimizer_update_l2))
and float(manifest.max_optimizer_update_l2) > 0.0
),
"loss_delta_finite": (
manifest.loss_delta is not None
and math.isfinite(float(manifest.loss_delta))
),
"updates_have_gradients": bool(gradient_l2_values)
and all(math.isfinite(value) and value > 0.0 for value in gradient_l2_values),
"updates_have_weight_delta": bool(update_l2_values)
and all(math.isfinite(value) and value > 0.0 for value in update_l2_values),
"proof_weight_sha256_ok": (
isinstance(proof.get("adapter_weight_sha256"), str)
and proof.get("adapter_weight_sha256") == weight_report["sha256"]
),
"proof_weight_stats_ok": (
proof_weight_stats.get("sha256") == weight_report["sha256"]
and proof_weight_stats.get("tensor_count") == weight_report["tensor_count"]
and proof_weight_stats.get("lora_tensor_count") == weight_report["lora_tensor_count"]
and proof_weight_stats.get("total_elements") == weight_report["total_elements"]
and proof_weight_stats.get("nonzero_elements") == weight_report["nonzero_elements"]
and int(proof_weight_stats.get("lora_tensor_count") or 0) > 0
and int(proof_weight_stats.get("nonzero_elements") or 0) > 0
and bool(proof_weight_stats.get("all_finite"))
),
"proof_cuda_bf16": (
bool(proof.get("bf16"))
and bool(proof_environment.get("cuda_available"))
and "RTX 4090" in str(proof_environment.get("device_name") or "")
),
"proof_bf16_supported": bool(proof_environment.get("bf16_supported")),
"proof_contract_ok": training_payload_contract_ok(proof, manifest),
"proof_optimizer_ok": proof.get("optimizer") == "paged_adamw_8bit",
"proof_max_length_ok": int(proof.get("max_length") or 0) >= 8192,
"proof_target_modules_ok": training_target_modules_ok(
manifest.kind,
proof.get("target_modules") if isinstance(proof.get("target_modules"), list) else [],
),
"proof_unsloth_runtime_ok": (
bool(proof.get("unsloth_version"))
and bool(proof.get("triton_version"))
and bool(proof.get("xformers_version"))
and bool(proof.get("unsloth_model_type"))
),
"proof_matches_manifest": (
proof.get("adapter_kind") == manifest.kind.value
and proof.get("proof_version") == manifest.proof_version
and proof.get("training_method") == manifest.training_method
and proof.get("precision") == manifest.precision
and proof.get("framework") == manifest.framework
and proof.get("base_model") == manifest.base_model
and proof.get("dataset_sha256") == manifest.dataset_sha256
and proof.get("sample_count") == manifest.sample_count
and proof.get("max_steps") == manifest.max_steps
and proof.get("max_length") == manifest.max_length
and proof.get("rank") == manifest.rank
and proof.get("alpha") == manifest.alpha
and proof.get("target_modules") == manifest.target_modules
and proof.get("optimizer") == manifest.optimizer
and proof.get("optimizer_family") == manifest.optimizer_family
and proof.get("learning_rate") == manifest.learning_rate
and proof.get("gradient_accumulation_steps") == manifest.gradient_accumulation_steps
and proof.get("gradient_checkpointing") == manifest.gradient_checkpointing
and proof.get("gradient_checkpointing_mode") == manifest.gradient_checkpointing_mode
and proof.get("autocast_dtype") == manifest.autocast_dtype
and proof.get("unsloth_version") == manifest.unsloth_version
and proof.get("triton_version") == manifest.triton_version
and proof.get("xformers_version") == manifest.xformers_version
and proof.get("unsloth_model_type") == manifest.unsloth_model_type
and proof.get("bf16") == manifest.bf16
and proof.get("trainable_params") == manifest.trainable_params
and proof.get("total_params") == manifest.total_params
and proof.get("initial_trainable_sha256") == manifest.initial_trainable_sha256
and proof.get("final_trainable_sha256") == manifest.final_trainable_sha256
and proof.get("optimizer_update_count") == manifest.optimizer_update_count
and proof.get("max_optimizer_update_l2") == manifest.max_optimizer_update_l2
),
}
)
report["valid"] = (
report["proof_sha256_ok"]
and report["hash_changed"]
and report["optimizer_update_count_ok"]
and report["max_optimizer_update_l2_ok"]
and report["loss_delta_finite"]
and report["updates_have_gradients"]
and report["updates_have_weight_delta"]
and report["proof_weight_sha256_ok"]
and report["proof_weight_stats_ok"]
and report["proof_cuda_bf16"]
and report["proof_bf16_supported"]
and report["proof_contract_ok"]
and report["proof_optimizer_ok"]
and report["proof_max_length_ok"]
and report["proof_target_modules_ok"]
and report["proof_unsloth_runtime_ok"]
and report["proof_matches_manifest"]
)
return report