diff --git a/backend/src/ely_eye/adapters.py b/backend/src/ely_eye/adapters.py index 78002d5..72c1df0 100644 --- a/backend/src/ely_eye/adapters.py +++ b/backend/src/ely_eye/adapters.py @@ -13,8 +13,14 @@ from pydantic import ValidationError from . import cartridge_store from .config import Settings from .db import Database -from .runtime_schemas import AdapterChoice, AdapterLoadReport, AdapterOrigin, AdapterState -from .schemas import AdapterManifest, TrainingAdapterKind +from .runtime_schemas import ( + AdapterChoice, + AdapterLoadReport, + AdapterManifest, + AdapterOrigin, + AdapterState, +) +from .schemas import TrainingAdapterKind logger = logging.getLogger("ely_eye.adapters") diff --git a/backend/src/ely_eye/api.py b/backend/src/ely_eye/api.py index 3ce7ce8..b2e3468 100644 --- a/backend/src/ely_eye/api.py +++ b/backend/src/ely_eye/api.py @@ -30,10 +30,8 @@ from .profiles import runtime_profile_report from .proofs import HashHopEvaluator, PRDProofSuite from .retrieval import RetrievalService from .runtime import QwenRuntime, RuntimeUnavailableError -from .runtime_schemas import AdapterLoadReport +from .runtime_schemas import AdapterLoadReport, AdapterManifest, AdapterStatus, RuntimeProfileReport from .schemas import ( - AdapterManifest, - AdapterStatus, AnswerResponse, AtomPage, EvidenceAtom, @@ -44,7 +42,6 @@ from .schemas import ( PathIngestRequest, RememberRequest, RuntimeProfile, - RuntimeProfileReport, SearchRequest, SearchResult, SupersedeRequest, diff --git a/backend/src/ely_eye/api_cartridges.py b/backend/src/ely_eye/api_cartridges.py index f8115bf..e0e3a88 100644 --- a/backend/src/ely_eye/api_cartridges.py +++ b/backend/src/ely_eye/api_cartridges.py @@ -12,7 +12,8 @@ from .cartridge import CartridgeService from .cartridge_assets import CartridgeAssetService from .config import Settings from .db import Database -from .schemas import AppendRequest, CartridgeAssetReport, CartridgeManifest, MountState +from .runtime_schemas import CartridgeAssetReport +from .schemas import AppendRequest, CartridgeManifest, MountState def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None: diff --git a/backend/src/ely_eye/cartridge.py b/backend/src/ely_eye/cartridge.py index 5f20545..2b9ea93 100644 --- a/backend/src/ely_eye/cartridge.py +++ b/backend/src/ely_eye/cartridge.py @@ -15,8 +15,8 @@ from . import cartridge_store from .config import Settings, get_settings from .db import Database from .kv_fingerprint import compute_fingerprint, fingerprint_diff, unverifiable_fields +from .runtime_schemas import AdapterManifest from .schemas import ( - AdapterManifest, CartridgeManifest, EvidenceAtom, MountState, diff --git a/backend/src/ely_eye/cartridge_assets.py b/backend/src/ely_eye/cartridge_assets.py index 4cbd1eb..a1780c9 100644 --- a/backend/src/ely_eye/cartridge_assets.py +++ b/backend/src/ely_eye/cartridge_assets.py @@ -17,7 +17,7 @@ from PIL import Image from .cartridge import JSON_PARQUET_COLUMNS, CartridgeService from .config import Settings, get_settings from .db import Database -from .schemas import CartridgeAssetReport, CartridgeAssetStatus +from .runtime_schemas import CartridgeAssetReport, CartridgeAssetStatus from .storage import ObjectStore, sha256_file TEXT_VECTOR_DIMS = 384 diff --git a/backend/src/ely_eye/profiles.py b/backend/src/ely_eye/profiles.py index bc04be7..bb3857a 100644 --- a/backend/src/ely_eye/profiles.py +++ b/backend/src/ely_eye/profiles.py @@ -7,7 +7,7 @@ import subprocess from pathlib import Path from .config import Settings -from .schemas import RuntimeLaunchProfile, RuntimeProfileReport, RuntimeProfileValidation +from .runtime_schemas import RuntimeLaunchProfile, RuntimeProfileReport, RuntimeProfileValidation EXTREME_YARN_OVERRIDE = { "text_config": { diff --git a/backend/src/ely_eye/proofs/checks_training.py b/backend/src/ely_eye/proofs/checks_training.py index 8914e0c..78e9e76 100644 --- a/backend/src/ely_eye/proofs/checks_training.py +++ b/backend/src/ely_eye/proofs/checks_training.py @@ -14,10 +14,8 @@ from ..db import Database from ..evaluation_schemas import ( ProofCheck, ) -from ..schemas import ( - AdapterManifest, - TrainingAdapterKind, -) +from ..runtime_schemas import AdapterManifest +from ..schemas import TrainingAdapterKind from ..training_common import sha256_file, sha256_tree from ..training_metrics import safetensor_weight_stats diff --git a/backend/src/ely_eye/proofs/training_trace.py b/backend/src/ely_eye/proofs/training_trace.py index f935d24..d13ad76 100644 --- a/backend/src/ely_eye/proofs/training_trace.py +++ b/backend/src/ely_eye/proofs/training_trace.py @@ -8,10 +8,8 @@ import math from pathlib import Path from typing import Any -from ..schemas import ( - AdapterManifest, - TrainingAdapterKind, -) +from ..runtime_schemas import AdapterManifest +from ..schemas import TrainingAdapterKind from ..training_common import ( AUTOCAST_DTYPE, GRADIENT_CHECKPOINTING_MODE, diff --git a/backend/src/ely_eye/runtime_schemas.py b/backend/src/ely_eye/runtime_schemas.py index 6c158d5..41d4de6 100644 --- a/backend/src/ely_eye/runtime_schemas.py +++ b/backend/src/ely_eye/runtime_schemas.py @@ -1,10 +1,13 @@ -"""Runtime observability shapes: what the model holds in memory right now.""" +"""Runtime and adapter shapes: what the model holds in memory, what trained it, and how a machine +launches it. A proof report moves when the evidence rules move; the memory model should not. +""" +from datetime import datetime from typing import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field -from .schemas import TrainingAdapterKind +from .schemas import TrainingAdapterKind, utc_now AdapterState = Literal["loaded", "available", "deferred", "missing"] AdapterOrigin = Literal["library", "cartridge"] @@ -28,3 +31,162 @@ class AdapterLoadReport(BaseModel): model_id: str mounted: list[str] adapters: list[AdapterChoice] + +class AdapterManifest(BaseModel): + adapter_id: str + kind: TrainingAdapterKind + created_at: datetime + proof_version: int = 1 + training_method: str = "BF16 LoRA" + precision: str = "bf16" + framework: str = "unsloth_patched_transformers_peft" + base_model: str + dataset_path: str + dataset_sha256: str + sample_count: int + max_steps: int + max_length: int + rank: int + alpha: int + target_modules: list[str] + trainable_params: int + total_params: int + optimizer: str + optimizer_family: str | None = None + learning_rate: float + gradient_accumulation_steps: int + gradient_checkpointing: bool = False + gradient_checkpointing_mode: str | None = None + autocast_dtype: str | None = None + unsloth_version: str | None = None + triton_version: str | None = None + xformers_version: str | None = None + unsloth_model_type: str | None = None + bf16: bool + final_loss: float | None + loss_history: list[float] = Field(default_factory=list) + initial_trainable_sha256: str | None = None + final_trainable_sha256: str | None = None + initial_trainable_l2: float | None = None + final_trainable_l2: float | None = None + optimizer_update_count: int = 0 + max_optimizer_update_l2: float | None = None + loss_delta: float | None = None + training_proof_path: str | None = None + training_proof_sha256: str | None = None + training_trace_path: str | None = None + training_trace_sha256: str | None = None + training_summary_path: str | None = None + training_summary_sha256: str | None = None + training_started_at: datetime | None = None + training_finished_at: datetime | None = None + training_wall_seconds: float | None = None + training_device: str | None = None + torch_version: str | None = None + cuda_version: str | None = None + adapter_weight_sha256: str | None = None + adapter_tensor_count: int = 0 + adapter_lora_tensor_count: int = 0 + adapter_total_elements: int = 0 + adapter_nonzero_elements: int = 0 + adapter_max_abs: float | None = None + adapter_weights_finite: bool = False + artifacts: dict[str, str] + sha256: str + cartridge_id: str | None = None + +class AdapterStatus(BaseModel): + name: str + kind: TrainingAdapterKind + proof_version: int = 1 + training_method: str = "BF16 LoRA" + precision: str = "bf16" + framework: str = "unsloth_patched_transformers_peft" + base_model: str + path: str + adapter_id: str + sample_count: int + max_steps: int + trainable_params: int + final_loss: float | None + sha256: str + weight_bytes: int + safetensor_keys: int + cartridge_id: str | None = None + cartridge_bound: bool + training_trace_path: str | None = None + training_trace_sha256: str | None = None + training_trace_steps: int = 0 + training_proof_path: str | None = None + training_proof_sha256: str | None = None + optimizer: str | None = None + optimizer_family: str | None = None + gradient_checkpointing: bool = False + gradient_checkpointing_mode: str | None = None + autocast_dtype: str | None = None + unsloth_version: str | None = None + triton_version: str | None = None + xformers_version: str | None = None + unsloth_model_type: str | None = None + optimizer_update_count: int = 0 + max_optimizer_update_l2: float | None = None + loss_delta: float | None = None + training_summary_path: str | None = None + training_wall_seconds: float | None = None + training_device: str | None = None + bf16: bool = False + torch_version: str | None = None + cuda_version: str | None = None + adapter_weight_sha256: str | None = None + adapter_tensor_count: int = 0 + adapter_lora_tensor_count: int = 0 + adapter_total_elements: int = 0 + adapter_nonzero_elements: int = 0 + adapter_max_abs: float | None = None + adapter_weights_finite: bool = False + +class CartridgeAssetStatus(BaseModel): + name: str + path: str + sha256: str + size_bytes: int + item_count: int + dtype: str + status: Literal["ready", "missing"] + +class CartridgeAssetReport(BaseModel): + cartridge_id: str + status: Literal["ready", "incomplete"] + created_at: datetime = Field(default_factory=utc_now) + assets: list[CartridgeAssetStatus] + +class RuntimeLaunchProfile(BaseModel): + name: str + runtime: str + role: str + context_tokens: int + script: str + command: list[str] + environment: dict[str, str] = Field(default_factory=dict) + required_features: list[str] = Field(default_factory=list) + +class RuntimeProfileValidation(BaseModel): + name: str + runtime: str + status: Literal["ready", "missing"] + script_exists: bool + package: str | None = None + package_version: str | None = None + executable: str | None = None + gpu_available: bool = False + probe_command: list[str] = Field(default_factory=list) + probe_status: Literal["passed", "failed", "skipped"] = "skipped" + probe_exit_code: int | None = None + probe_output: str | None = None + detail: str + +class RuntimeProfileReport(BaseModel): + status: Literal["ready", "incomplete"] + created_at: datetime = Field(default_factory=utc_now) + profiles: list[RuntimeLaunchProfile] + validations: list[RuntimeProfileValidation] = Field(default_factory=list) diff --git a/backend/src/ely_eye/schemas.py b/backend/src/ely_eye/schemas.py index a1083ec..31495ef 100644 --- a/backend/src/ely_eye/schemas.py +++ b/backend/src/ely_eye/schemas.py @@ -266,165 +266,6 @@ class CartridgeManifest(BaseModel): kv_fingerprint: KvFingerprint | None = None dna: str | None = None -class AdapterManifest(BaseModel): - adapter_id: str - kind: TrainingAdapterKind - created_at: datetime - proof_version: int = 1 - training_method: str = "BF16 LoRA" - precision: str = "bf16" - framework: str = "unsloth_patched_transformers_peft" - base_model: str - dataset_path: str - dataset_sha256: str - sample_count: int - max_steps: int - max_length: int - rank: int - alpha: int - target_modules: list[str] - trainable_params: int - total_params: int - optimizer: str - optimizer_family: str | None = None - learning_rate: float - gradient_accumulation_steps: int - gradient_checkpointing: bool = False - gradient_checkpointing_mode: str | None = None - autocast_dtype: str | None = None - unsloth_version: str | None = None - triton_version: str | None = None - xformers_version: str | None = None - unsloth_model_type: str | None = None - bf16: bool - final_loss: float | None - loss_history: list[float] = Field(default_factory=list) - initial_trainable_sha256: str | None = None - final_trainable_sha256: str | None = None - initial_trainable_l2: float | None = None - final_trainable_l2: float | None = None - optimizer_update_count: int = 0 - max_optimizer_update_l2: float | None = None - loss_delta: float | None = None - training_proof_path: str | None = None - training_proof_sha256: str | None = None - training_trace_path: str | None = None - training_trace_sha256: str | None = None - training_summary_path: str | None = None - training_summary_sha256: str | None = None - training_started_at: datetime | None = None - training_finished_at: datetime | None = None - training_wall_seconds: float | None = None - training_device: str | None = None - torch_version: str | None = None - cuda_version: str | None = None - adapter_weight_sha256: str | None = None - adapter_tensor_count: int = 0 - adapter_lora_tensor_count: int = 0 - adapter_total_elements: int = 0 - adapter_nonzero_elements: int = 0 - adapter_max_abs: float | None = None - adapter_weights_finite: bool = False - artifacts: dict[str, str] - sha256: str - cartridge_id: str | None = None - -class AdapterStatus(BaseModel): - name: str - kind: TrainingAdapterKind - proof_version: int = 1 - training_method: str = "BF16 LoRA" - precision: str = "bf16" - framework: str = "unsloth_patched_transformers_peft" - base_model: str - path: str - adapter_id: str - sample_count: int - max_steps: int - trainable_params: int - final_loss: float | None - sha256: str - weight_bytes: int - safetensor_keys: int - cartridge_id: str | None = None - cartridge_bound: bool - training_trace_path: str | None = None - training_trace_sha256: str | None = None - training_trace_steps: int = 0 - training_proof_path: str | None = None - training_proof_sha256: str | None = None - optimizer: str | None = None - optimizer_family: str | None = None - gradient_checkpointing: bool = False - gradient_checkpointing_mode: str | None = None - autocast_dtype: str | None = None - unsloth_version: str | None = None - triton_version: str | None = None - xformers_version: str | None = None - unsloth_model_type: str | None = None - optimizer_update_count: int = 0 - max_optimizer_update_l2: float | None = None - loss_delta: float | None = None - training_summary_path: str | None = None - training_wall_seconds: float | None = None - training_device: str | None = None - bf16: bool = False - torch_version: str | None = None - cuda_version: str | None = None - adapter_weight_sha256: str | None = None - adapter_tensor_count: int = 0 - adapter_lora_tensor_count: int = 0 - adapter_total_elements: int = 0 - adapter_nonzero_elements: int = 0 - adapter_max_abs: float | None = None - adapter_weights_finite: bool = False - -class CartridgeAssetStatus(BaseModel): - name: str - path: str - sha256: str - size_bytes: int - item_count: int - dtype: str - status: Literal["ready", "missing"] - -class CartridgeAssetReport(BaseModel): - cartridge_id: str - status: Literal["ready", "incomplete"] - created_at: datetime = Field(default_factory=utc_now) - assets: list[CartridgeAssetStatus] - -class RuntimeLaunchProfile(BaseModel): - name: str - runtime: str - role: str - context_tokens: int - script: str - command: list[str] - environment: dict[str, str] = Field(default_factory=dict) - required_features: list[str] = Field(default_factory=list) - -class RuntimeProfileValidation(BaseModel): - name: str - runtime: str - status: Literal["ready", "missing"] - script_exists: bool - package: str | None = None - package_version: str | None = None - executable: str | None = None - gpu_available: bool = False - probe_command: list[str] = Field(default_factory=list) - probe_status: Literal["passed", "failed", "skipped"] = "skipped" - probe_exit_code: int | None = None - probe_output: str | None = None - detail: str - -class RuntimeProfileReport(BaseModel): - status: Literal["ready", "incomplete"] - created_at: datetime = Field(default_factory=utc_now) - profiles: list[RuntimeLaunchProfile] - validations: list[RuntimeProfileValidation] = Field(default_factory=list) - class ForgetReport(BaseModel): """What a forget actually removed, including the derived copies that have to go with it.""" diff --git a/backend/src/ely_eye/training.py b/backend/src/ely_eye/training.py index 171eb78..5c9d0b2 100644 --- a/backend/src/ely_eye/training.py +++ b/backend/src/ely_eye/training.py @@ -6,7 +6,8 @@ from pathlib import Path from .config import Settings, get_settings from .db import Database -from .schemas import AdapterManifest, TrainingAdapterKind +from .runtime_schemas import AdapterManifest +from .schemas import TrainingAdapterKind from .training_common import TrainingSample from .training_datasets import ( build_prd_dataset, diff --git a/backend/src/ely_eye/training_lora.py b/backend/src/ely_eye/training_lora.py index 97344ea..4dc4e92 100644 --- a/backend/src/ely_eye/training_lora.py +++ b/backend/src/ely_eye/training_lora.py @@ -10,7 +10,8 @@ from typing import Any from .config import Settings from .db import Database -from .schemas import AdapterManifest, TrainingAdapterKind +from .runtime_schemas import AdapterManifest +from .schemas import TrainingAdapterKind from .training_common import ( AUTOCAST_DTYPE, GRADIENT_CHECKPOINTING_MODE, diff --git a/backend/src/ely_eye/training_retrieval.py b/backend/src/ely_eye/training_retrieval.py index 7aaba1b..d4dd161 100644 --- a/backend/src/ely_eye/training_retrieval.py +++ b/backend/src/ely_eye/training_retrieval.py @@ -9,7 +9,8 @@ from typing import Any from .config import Settings from .db import Database -from .schemas import AdapterManifest, TrainingAdapterKind +from .runtime_schemas import AdapterManifest +from .schemas import TrainingAdapterKind from .training_common import ( AUTOCAST_DTYPE, GRADIENT_CHECKPOINTING_MODE,