Resolve adapters by what is mounted, and say why each kind is not loaded
runtime.py resolved adapters at 7 call sites and only ever looked in the
adapter library, so an adapter attached to a cartridge (cartridge.py:230
copies it into <cartridge>/adapters/{kind}/) was never loaded by anything.
Resolution now lives in adapters.py: it reads the mounted cartridge set,
prefers a mounted cartridge's copy over the library, and answers every one
of the six kinds with a state the facts support — loaded, available,
deferred (base_model mismatch or missing weights, both models named in the
reason), missing — plus a shadowed row for each adapter that lost the
choice. A mounted cartridge is the scenario selection; neither manifest
carries a scenario field, so none was invented.
Two bugs the new tests caught: the preference sort lost reverse=True, so a
library adapter beat a mounted cartridge's copy, and the load branch had
never been executed by a test. Stub weights now prove peft is asked to
attach, that the returned model is put in eval mode, and that the endpoint
reports loaded until unload clears it.
Faces: GET /api/runtime/adapters, `ely-eye adapters`, and a log line naming
the reason whenever a generation runs without an adapter.
Gates: 222 passed; ruff clean; mypy cold cache on the 7 touched files gives
the same 2 errors as HEAD's runtime.py, and pyright gives the same 2 as
HEAD — both are qwen_vl_utils typing artifacts (process_vision_info returns
2 or 3 values depending on return_video_kwargs; the product takes 2).
This commit is contained in:
1 parent
9f0f7993e5
commit
2fbb4af96b
8 files changed
+585
-86
No files matched your search
@@ -0,0 +1,229 @@
|
||||
"""One place decides which adapter loads, and says out loud why the others do not."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from . import cartridge_store
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .runtime_schemas import AdapterChoice, AdapterLoadReport, AdapterOrigin, AdapterState
|
||||
from .schemas import AdapterManifest, TrainingAdapterKind
|
||||
|
||||
logger = logging.getLogger("ely_eye.adapters")
|
||||
|
||||
MANIFEST_FILE = "adapter_manifest.json"
|
||||
WEIGHTS_FILE = "adapter_model.safetensors"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AdapterSource:
|
||||
"""One adapter found on disk, plus where it was found."""
|
||||
|
||||
path: Path
|
||||
manifest: AdapterManifest
|
||||
origin: AdapterOrigin
|
||||
cartridge_id: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoadRecord:
|
||||
"""An adapter this process actually attached to a live model."""
|
||||
|
||||
kind: TrainingAdapterKind
|
||||
adapter_id: str
|
||||
path: str
|
||||
sha256: str
|
||||
loaded_at: str
|
||||
|
||||
|
||||
_LOADED: dict[str, LoadRecord] = {}
|
||||
|
||||
|
||||
def read_manifest(path: Path) -> AdapterManifest:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except OSError as error:
|
||||
raise ValueError(f"{path}: the adapter manifest cannot be read ({error})") from error
|
||||
try:
|
||||
return AdapterManifest.model_validate_json(text)
|
||||
except ValidationError as error:
|
||||
raise ValueError(f"{path}: the adapter manifest is not a valid AdapterManifest ({error})") from error
|
||||
|
||||
|
||||
def cartridge_roots(database: Database) -> dict[str, Path]:
|
||||
roots: dict[str, Path] = {}
|
||||
for row in database.list_cartridges():
|
||||
roots[str(row["cartridge_id"])] = Path(str(row["root_path"]))
|
||||
return roots
|
||||
|
||||
|
||||
def library_sources(settings: Settings) -> list[AdapterSource]:
|
||||
"""Adapters in the library directory, for callers that have no database open."""
|
||||
if settings.adapter_dir is not None:
|
||||
pinned = settings.adapter_dir.expanduser()
|
||||
if not (pinned / MANIFEST_FILE).exists():
|
||||
logger.warning("adapter_dir %s holds no %s", pinned, MANIFEST_FILE)
|
||||
return []
|
||||
manifest = read_manifest(pinned / MANIFEST_FILE)
|
||||
return [
|
||||
AdapterSource(
|
||||
path=pinned,
|
||||
manifest=manifest,
|
||||
origin="library",
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
)
|
||||
]
|
||||
sources: list[AdapterSource] = []
|
||||
for manifest_path in sorted(settings.adapters_dir.glob(f"*/{MANIFEST_FILE}")):
|
||||
manifest = read_manifest(manifest_path)
|
||||
sources.append(
|
||||
AdapterSource(
|
||||
path=manifest_path.parent,
|
||||
manifest=manifest,
|
||||
origin="library",
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
)
|
||||
)
|
||||
return sources
|
||||
|
||||
|
||||
def discover(settings: Settings, database: Database) -> tuple[list[AdapterSource], list[str]]:
|
||||
"""Every adapter the runtime could use: the library plus the copies inside mounted cartridges."""
|
||||
mounted = cartridge_store.mounted_ids(database)
|
||||
roots = cartridge_roots(database)
|
||||
sources = library_sources(settings)
|
||||
for cartridge_id in mounted:
|
||||
root = roots.get(cartridge_id)
|
||||
if root is None:
|
||||
logger.warning("cartridge %s is mounted but its directory is not registered", cartridge_id)
|
||||
continue
|
||||
for manifest_path in sorted((root / "adapters").glob(f"*/{MANIFEST_FILE}")):
|
||||
sources.append(
|
||||
AdapterSource(
|
||||
path=manifest_path.parent,
|
||||
manifest=read_manifest(manifest_path),
|
||||
origin="cartridge",
|
||||
cartridge_id=cartridge_id,
|
||||
)
|
||||
)
|
||||
return sources, mounted
|
||||
|
||||
|
||||
def resolve(
|
||||
sources: list[AdapterSource],
|
||||
settings: Settings,
|
||||
kind: TrainingAdapterKind,
|
||||
) -> tuple[AdapterSource | None, AdapterState, str]:
|
||||
"""Pick the adapter to load for one kind, or say which state it is stuck in and why."""
|
||||
candidates = [source for source in sources if source.manifest.kind == kind]
|
||||
if not candidates:
|
||||
return None, "missing", f"no adapter of kind {kind.value} is on disk"
|
||||
matching = [source for source in candidates if source.manifest.base_model == settings.model_id]
|
||||
if not matching:
|
||||
trained_on = ", ".join(sorted({source.manifest.base_model for source in candidates}))
|
||||
return (
|
||||
None,
|
||||
"deferred",
|
||||
f"every {kind.value} adapter is trained on {trained_on}; the runtime holds {settings.model_id}",
|
||||
)
|
||||
ready = [source for source in matching if (source.path / WEIGHTS_FILE).exists()]
|
||||
if not ready:
|
||||
return None, "deferred", f"the {kind.value} manifests are present but none carries {WEIGHTS_FILE}"
|
||||
ready.sort(key=_preference, reverse=True)
|
||||
chosen = ready[0]
|
||||
if chosen.origin == "cartridge":
|
||||
return chosen, "available", f"selected from mounted cartridge {chosen.cartridge_id}"
|
||||
return chosen, "available", "selected from the adapter library"
|
||||
|
||||
|
||||
def resolve_library_adapter(
|
||||
settings: Settings,
|
||||
kind: TrainingAdapterKind = TrainingAdapterKind.hme_core_lora,
|
||||
) -> tuple[AdapterSource | None, AdapterState, str]:
|
||||
"""Resolution without a database: the library only, no mounted cartridges."""
|
||||
return resolve(library_sources(settings), settings, kind)
|
||||
|
||||
|
||||
def _preference(source: AdapterSource) -> tuple[bool, float]:
|
||||
weights = source.path / WEIGHTS_FILE
|
||||
return (source.origin == "cartridge", weights.stat().st_mtime if weights.exists() else 0.0)
|
||||
|
||||
|
||||
def load_records() -> list[LoadRecord]:
|
||||
return list(_LOADED.values())
|
||||
|
||||
|
||||
def clear_loads() -> None:
|
||||
_LOADED.clear()
|
||||
|
||||
|
||||
def record_load(source: AdapterSource) -> LoadRecord:
|
||||
record = LoadRecord(
|
||||
kind=source.manifest.kind,
|
||||
adapter_id=source.manifest.adapter_id,
|
||||
path=str(source.path),
|
||||
sha256=source.manifest.sha256,
|
||||
loaded_at=datetime.now().astimezone().isoformat(),
|
||||
)
|
||||
_LOADED[record.kind.value] = record
|
||||
logger.info("loaded adapter %s (%s) from %s", record.adapter_id, record.kind.value, record.path)
|
||||
return record
|
||||
|
||||
|
||||
def load_into_model(model: Any, path: Path) -> Any:
|
||||
"""Attach one LoRA to a live model; the only place generation imports peft."""
|
||||
from .training import bitsandbytes_gaudi_probe_guard
|
||||
|
||||
with bitsandbytes_gaudi_probe_guard():
|
||||
from peft import PeftModel
|
||||
|
||||
return PeftModel.from_pretrained(model, str(path))
|
||||
|
||||
|
||||
def plan(settings: Settings, database: Database) -> AdapterLoadReport:
|
||||
"""One row per adapter kind, plus a row for every adapter that lost the choice."""
|
||||
sources, mounted = discover(settings, database)
|
||||
rows: list[AdapterChoice] = []
|
||||
for kind in TrainingAdapterKind:
|
||||
source, state, reason = resolve(sources, settings, kind)
|
||||
record = next((item for item in load_records() if item.kind == kind), None)
|
||||
if record is not None:
|
||||
reason = f"attached to the live model from {record.path}"
|
||||
state = "loaded"
|
||||
rows.append(
|
||||
AdapterChoice(
|
||||
kind=kind,
|
||||
state=state,
|
||||
reason=reason,
|
||||
path=str(source.path) if source else None,
|
||||
adapter_id=source.manifest.adapter_id if source else None,
|
||||
base_model=source.manifest.base_model if source else settings.model_id,
|
||||
origin=source.origin if source else None,
|
||||
cartridge_id=source.cartridge_id if source else None,
|
||||
)
|
||||
)
|
||||
chosen_paths = {row.path for row in rows if row.path}
|
||||
for source in sources:
|
||||
if str(source.path) in chosen_paths:
|
||||
continue
|
||||
rows.append(
|
||||
AdapterChoice(
|
||||
kind=source.manifest.kind,
|
||||
state="deferred",
|
||||
reason="another adapter of this kind wins for the current mount",
|
||||
path=str(source.path),
|
||||
adapter_id=source.manifest.adapter_id,
|
||||
base_model=source.manifest.base_model,
|
||||
origin=source.origin,
|
||||
cartridge_id=source.cartridge_id,
|
||||
shadowed=True,
|
||||
)
|
||||
)
|
||||
return AdapterLoadReport(model_id=settings.model_id, mounted=mounted, adapters=rows)
|
||||
@@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
|
||||
from . import __version__
|
||||
from .adapters import plan as adapter_plan
|
||||
from .api_cartridges import register as register_cartridge_routes
|
||||
from .api_holders import register as register_holder_routes
|
||||
from .benchmarks import (
|
||||
@@ -41,6 +42,7 @@ from .proofs import HashHopEvaluator, PRDProofSuite
|
||||
from .research_modules import ResearchModuleService
|
||||
from .retrieval import RetrievalService
|
||||
from .runtime import QwenRuntime, RuntimeUnavailableError
|
||||
from .runtime_schemas import AdapterLoadReport
|
||||
from .schemas import (
|
||||
AdapterManifest,
|
||||
AdapterStatus,
|
||||
@@ -277,6 +279,10 @@ def create_app() -> FastAPI:
|
||||
def runtime_profiles() -> RuntimeProfileReport:
|
||||
return runtime_profile_report()
|
||||
|
||||
@app.get("/api/runtime/adapters", response_model=AdapterLoadReport)
|
||||
def runtime_adapters(settings: Annotated[Settings, Depends(settings_dep)]) -> AdapterLoadReport:
|
||||
return adapter_plan(settings, Database(settings))
|
||||
|
||||
@app.get("/api/research/modules")
|
||||
def research_modules(settings: Annotated[Settings, Depends(settings_dep)]):
|
||||
db = Database(settings)
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
import typer
|
||||
import uvicorn
|
||||
|
||||
from .adapters import plan as adapter_plan
|
||||
from .api import create_app
|
||||
from .benchmarks import (
|
||||
benchmark_prediction_report,
|
||||
@@ -475,5 +476,13 @@ def attach_adapter(cartridge_id: str, adapter_dir: Path) -> None:
|
||||
typer.echo(manifest.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command("adapters")
|
||||
def adapter_status() -> None:
|
||||
"""Which adapter the runtime would load, and why each other kind would not."""
|
||||
settings = get_settings()
|
||||
report = adapter_plan(settings, Database(settings))
|
||||
typer.echo(json_dump(report.model_dump(mode="json")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .. import adapters
|
||||
from ..compiler import ContextCompiler
|
||||
from ..config import Settings
|
||||
from ..db import Database
|
||||
@@ -17,7 +18,7 @@ from ..evaluation_schemas import (
|
||||
)
|
||||
from ..evidence_graph import EvidenceGraphService
|
||||
from ..profiles import runtime_profile_report
|
||||
from ..runtime import QwenRuntime, resolve_runtime_adapter
|
||||
from ..runtime import QwenRuntime
|
||||
from ..schemas import (
|
||||
RuntimeProfile,
|
||||
TrainingAdapterKind,
|
||||
@@ -58,7 +59,7 @@ class RuntimeChecks:
|
||||
return deferred
|
||||
runtime = QwenRuntime(self.settings)
|
||||
status = runtime.status()
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
adapter, _adapter_state, _adapter_reason = adapters.resolve_library_adapter(self.settings)
|
||||
weight_path = adapter.path / "adapter_model.safetensors" if adapter else None
|
||||
payload = {
|
||||
"runtime_status": status.model_dump(mode="json"),
|
||||
@@ -133,7 +134,7 @@ class RuntimeChecks:
|
||||
)
|
||||
payload["generation_question"] = generation_question
|
||||
status = runtime.status()
|
||||
adapter = resolve_runtime_adapter(proof_settings)
|
||||
adapter, _adapter_state, _adapter_reason = adapters.resolve_library_adapter(proof_settings)
|
||||
generation = runtime.generate(generation_question, context)
|
||||
answer, citations, citation_report = Verifier().verify_answer(generation.text, context)
|
||||
claim_id = EvidenceGraphService(proof_settings, self.db).record_answer(
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -9,10 +10,14 @@ from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from . import adapters
|
||||
from .config import Settings, get_settings
|
||||
from .schemas import AdapterManifest, CompiledContext, RuntimeStatus, TrainingAdapterKind
|
||||
from .db import Database
|
||||
from .schemas import CompiledContext, RuntimeStatus, TrainingAdapterKind
|
||||
from .storage import ObjectStore
|
||||
|
||||
logger = logging.getLogger("ely_eye.runtime")
|
||||
|
||||
|
||||
class RuntimeUnavailableError(RuntimeError):
|
||||
pass
|
||||
@@ -34,12 +39,41 @@ class QwenRuntime:
|
||||
def __init__(self, settings: Settings | None = None) -> None:
|
||||
self.settings = settings or get_settings()
|
||||
self.objects = ObjectStore(self.settings)
|
||||
self.database = Database(self.settings)
|
||||
self._processor: Any | None = None
|
||||
self._model: Any | None = None
|
||||
|
||||
def _adapter_choice(self) -> tuple[adapters.AdapterSource | None, str]:
|
||||
"""The adapter this runtime would attach, plus the reason if it would not attach one."""
|
||||
sources, _ = adapters.discover(self.settings, self.database)
|
||||
source, _state, reason = adapters.resolve(sources, self.settings, TrainingAdapterKind.hme_core_lora)
|
||||
return source, reason
|
||||
|
||||
def _loaded_adapter(self) -> adapters.LoadRecord | None:
|
||||
return next((item for item in adapters.load_records() if item.kind == TrainingAdapterKind.hme_core_lora), None)
|
||||
|
||||
def _generation_adapter_fields(self) -> dict[str, str | None]:
|
||||
record = self._loaded_adapter()
|
||||
if record is not None:
|
||||
return {
|
||||
"adapter_path": record.path,
|
||||
"adapter_kind": record.kind.value,
|
||||
"adapter_id": record.adapter_id,
|
||||
"adapter_sha256": record.sha256,
|
||||
}
|
||||
source, _ = self._adapter_choice()
|
||||
if source is None:
|
||||
return {"adapter_path": None, "adapter_kind": None, "adapter_id": None, "adapter_sha256": None}
|
||||
return {
|
||||
"adapter_path": str(source.path),
|
||||
"adapter_kind": source.manifest.kind.value,
|
||||
"adapter_id": source.manifest.adapter_id,
|
||||
"adapter_sha256": source.manifest.sha256,
|
||||
}
|
||||
|
||||
def status(self) -> RuntimeStatus:
|
||||
gpu = gpu_status()
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
fields = self._generation_adapter_fields()
|
||||
if self.settings.runtime_backend == "sglang":
|
||||
available, detail = self._sglang_status()
|
||||
else:
|
||||
@@ -49,10 +83,10 @@ class QwenRuntime:
|
||||
model_id=self.settings.model_id,
|
||||
available=available,
|
||||
detail=detail,
|
||||
adapter_path=str(adapter.path) if adapter else None,
|
||||
adapter_kind=adapter.manifest.kind if adapter else None,
|
||||
adapter_id=adapter.manifest.adapter_id if adapter else None,
|
||||
adapter_sha256=adapter.manifest.sha256 if adapter else None,
|
||||
adapter_path=fields["adapter_path"],
|
||||
adapter_kind=TrainingAdapterKind(fields["adapter_kind"]) if fields["adapter_kind"] else None,
|
||||
adapter_id=fields["adapter_id"],
|
||||
adapter_sha256=fields["adapter_sha256"],
|
||||
gpu_name=gpu.get("name"),
|
||||
cuda=gpu.get("cuda"),
|
||||
vram_total_mb=gpu.get("total_mb"),
|
||||
@@ -84,15 +118,17 @@ class QwenRuntime:
|
||||
import torch
|
||||
|
||||
adapter_detail = ""
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
if adapter is not None:
|
||||
adapter_detail = f" Adapter selected: {adapter.path}."
|
||||
source, reason = self._adapter_choice()
|
||||
if source is not None:
|
||||
adapter_detail = f" Adapter selected: {source.path}."
|
||||
elif self.settings.adapter_dir is not None:
|
||||
return False, f"Configured adapter path is missing: {self.settings.adapter_dir}"
|
||||
if source is None:
|
||||
adapter_detail = f" No adapter: {reason}."
|
||||
if torch.cuda.is_available():
|
||||
return True, f"Transformers backend can use CUDA.{adapter_detail}"
|
||||
return True, f"Transformers backend is available on CPU.{adapter_detail}"
|
||||
except Exception as exc:
|
||||
except Exception as exc: # noqa: BLE001 # a status probe reports why the backend is unusable
|
||||
return False, f"Transformers backend import failed: {exc}"
|
||||
|
||||
def _generate_sglang(self, question: str, context: CompiledContext) -> RuntimeGeneration:
|
||||
@@ -122,15 +158,11 @@ class QwenRuntime:
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeUnavailableError(f"SGLang generation failed: {exc}") from exc
|
||||
data = response.json()
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
usage = data.get("usage") if isinstance(data, dict) else None
|
||||
return RuntimeGeneration(
|
||||
text=data["choices"][0]["message"]["content"],
|
||||
backend="sglang",
|
||||
adapter_path=str(adapter.path) if adapter else None,
|
||||
adapter_kind=adapter.manifest.kind.value if adapter else None,
|
||||
adapter_id=adapter.manifest.adapter_id if adapter else None,
|
||||
adapter_sha256=adapter.manifest.sha256 if adapter else None,
|
||||
**self._generation_adapter_fields(),
|
||||
input_tokens=usage.get("prompt_tokens") if isinstance(usage, dict) else None,
|
||||
output_tokens=usage.get("completion_tokens") if isinstance(usage, dict) else None,
|
||||
)
|
||||
@@ -161,15 +193,11 @@ class QwenRuntime:
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeUnavailableError(f"SGLang generation failed: {exc}") from exc
|
||||
data = response.json()
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
usage = data.get("usage") if isinstance(data, dict) else None
|
||||
return RuntimeGeneration(
|
||||
text=data["choices"][0]["message"]["content"],
|
||||
backend="sglang",
|
||||
adapter_path=str(adapter.path) if adapter else None,
|
||||
adapter_kind=adapter.manifest.kind.value if adapter else None,
|
||||
adapter_id=adapter.manifest.adapter_id if adapter else None,
|
||||
adapter_sha256=adapter.manifest.sha256 if adapter else None,
|
||||
**self._generation_adapter_fields(),
|
||||
input_tokens=usage.get("prompt_tokens") if isinstance(usage, dict) else None,
|
||||
output_tokens=usage.get("completion_tokens") if isinstance(usage, dict) else None,
|
||||
)
|
||||
@@ -218,14 +246,10 @@ class QwenRuntime:
|
||||
input_length = int(inputs["input_ids"].shape[-1])
|
||||
generated = generated[:, input_length:]
|
||||
decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
return RuntimeGeneration(
|
||||
text=decoded,
|
||||
backend="transformers",
|
||||
adapter_path=str(adapter.path) if adapter else None,
|
||||
adapter_kind=adapter.manifest.kind.value if adapter else None,
|
||||
adapter_id=adapter.manifest.adapter_id if adapter else None,
|
||||
adapter_sha256=adapter.manifest.sha256 if adapter else None,
|
||||
**self._generation_adapter_fields(),
|
||||
input_tokens=input_length,
|
||||
output_tokens=int(generated.shape[-1]),
|
||||
)
|
||||
@@ -273,14 +297,10 @@ class QwenRuntime:
|
||||
input_length = int(inputs["input_ids"].shape[-1])
|
||||
generated = generated[:, input_length:]
|
||||
decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
return RuntimeGeneration(
|
||||
text=decoded,
|
||||
backend="transformers",
|
||||
adapter_path=str(adapter.path) if adapter else None,
|
||||
adapter_kind=adapter.manifest.kind.value if adapter else None,
|
||||
adapter_id=adapter.manifest.adapter_id if adapter else None,
|
||||
adapter_sha256=adapter.manifest.sha256 if adapter else None,
|
||||
**self._generation_adapter_fields(),
|
||||
input_tokens=input_length,
|
||||
output_tokens=int(generated.shape[-1]),
|
||||
)
|
||||
@@ -288,6 +308,19 @@ class QwenRuntime:
|
||||
def _load_transformers(self) -> tuple[Any, Any]:
|
||||
if self._model is not None and self._processor is not None:
|
||||
return self._model, self._processor
|
||||
model, processor = self._load_base_model()
|
||||
source, reason = self._adapter_choice()
|
||||
if source is None:
|
||||
logger.info("generating without an adapter: %s", reason)
|
||||
else:
|
||||
model = adapters.load_into_model(model, source.path)
|
||||
adapters.record_load(source)
|
||||
model.eval()
|
||||
self._model, self._processor = model, processor
|
||||
return model, processor
|
||||
|
||||
def _load_base_model(self) -> tuple[Any, Any]:
|
||||
"""Load the chat model; tests replace this so the adapter branch runs without a 9B download."""
|
||||
import torch
|
||||
from transformers import AutoProcessor
|
||||
|
||||
@@ -300,33 +333,25 @@ class QwenRuntime:
|
||||
|
||||
model_cls = AutoModelForCausalLM
|
||||
|
||||
self._processor = AutoProcessor.from_pretrained(self.settings.model_id, trust_remote_code=True)
|
||||
self._model = model_cls.from_pretrained(
|
||||
processor = AutoProcessor.from_pretrained(self.settings.model_id, trust_remote_code=True)
|
||||
model = model_cls.from_pretrained(
|
||||
self.settings.model_id,
|
||||
dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
|
||||
device_map="auto",
|
||||
trust_remote_code=True,
|
||||
)
|
||||
adapter = resolve_runtime_adapter(self.settings)
|
||||
if adapter is not None:
|
||||
from .training import bitsandbytes_gaudi_probe_guard
|
||||
|
||||
with bitsandbytes_gaudi_probe_guard():
|
||||
from peft import PeftModel
|
||||
|
||||
self._model = PeftModel.from_pretrained(self._model, adapter.path)
|
||||
self._model.eval()
|
||||
return self._model, self._processor
|
||||
return model, processor
|
||||
|
||||
def unload(self) -> None:
|
||||
self._model = None
|
||||
self._processor = None
|
||||
adapters.clear_loads()
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 # freeing the cache is best-effort, unloading must not raise
|
||||
return
|
||||
|
||||
def _context_images(self, context: CompiledContext) -> list[Path]:
|
||||
@@ -338,43 +363,6 @@ class QwenRuntime:
|
||||
return paths
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeAdapter:
|
||||
path: Path
|
||||
manifest: AdapterManifest
|
||||
|
||||
|
||||
def resolve_runtime_adapter(
|
||||
settings: Settings,
|
||||
kind: TrainingAdapterKind = TrainingAdapterKind.hme_core_lora,
|
||||
) -> RuntimeAdapter | None:
|
||||
if settings.adapter_dir is not None:
|
||||
path = settings.adapter_dir.expanduser().resolve()
|
||||
manifest_path = path / "adapter_manifest.json"
|
||||
if path.exists() and manifest_path.exists():
|
||||
return RuntimeAdapter(
|
||||
path=path,
|
||||
manifest=AdapterManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")),
|
||||
)
|
||||
return None
|
||||
|
||||
candidates: list[RuntimeAdapter] = []
|
||||
for manifest_path in settings.adapters_dir.glob("*/adapter_manifest.json"):
|
||||
manifest = AdapterManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest.kind == kind and manifest.base_model == settings.model_id:
|
||||
candidates.append(RuntimeAdapter(path=manifest_path.parent.resolve(), manifest=manifest))
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
item.manifest.cartridge_id is not None,
|
||||
(item.path / "adapter_model.safetensors").stat().st_mtime
|
||||
if (item.path / "adapter_model.safetensors").exists()
|
||||
else 0.0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def build_prompt(question: str, context: CompiledContext) -> str:
|
||||
contract = "\n".join(f"- {item}" for item in context.plan.verifier_contract)
|
||||
return f"""You are Ely-Eye, a local multimodal context operating system.
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Runtime observability shapes: what the model holds in memory right now."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .schemas import TrainingAdapterKind
|
||||
|
||||
AdapterState = Literal["loaded", "available", "deferred", "missing"]
|
||||
AdapterOrigin = Literal["library", "cartridge"]
|
||||
|
||||
|
||||
class AdapterChoice(BaseModel):
|
||||
"""One adapter kind, and the honest reason it is or is not attached to the model."""
|
||||
|
||||
kind: TrainingAdapterKind
|
||||
state: AdapterState
|
||||
reason: str
|
||||
path: str | None = None
|
||||
adapter_id: str | None = None
|
||||
base_model: str | None = None
|
||||
origin: AdapterOrigin | None = None
|
||||
cartridge_id: str | None = None
|
||||
shadowed: bool = False
|
||||
|
||||
|
||||
class AdapterLoadReport(BaseModel):
|
||||
model_id: str
|
||||
mounted: list[str]
|
||||
adapters: list[AdapterChoice]
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Which adapter the runtime loads, and the reason every other kind does not."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from ely_eye import adapters, api
|
||||
from ely_eye.cartridge_store import set_mounts
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.runtime import QwenRuntime
|
||||
from ely_eye.schemas import CartridgeManifest, TrainingAdapterKind
|
||||
|
||||
CORE = TrainingAdapterKind.hme_core_lora
|
||||
MODEL = "Qwen/Qwen3.5-9B"
|
||||
|
||||
|
||||
def _manifest(kind: TrainingAdapterKind, base_model: str, adapter_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"adapter_id": adapter_id,
|
||||
"kind": kind.value,
|
||||
"created_at": "2026-05-01T00:00:00+00:00",
|
||||
"base_model": base_model,
|
||||
"dataset_path": "PRD.md",
|
||||
"dataset_sha256": "d" * 64,
|
||||
"sample_count": 8,
|
||||
"max_steps": 16,
|
||||
"max_length": 512,
|
||||
"rank": 8,
|
||||
"alpha": 16,
|
||||
"target_modules": ["q_proj"],
|
||||
"trainable_params": 1_000,
|
||||
"total_params": 10_000,
|
||||
"optimizer": "adamw_8bit",
|
||||
"learning_rate": 0.0002,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"bf16": True,
|
||||
"final_loss": 0.01,
|
||||
"artifacts": {},
|
||||
"sha256": "a" * 64,
|
||||
}
|
||||
|
||||
|
||||
def _write_adapter(
|
||||
root: Path,
|
||||
kind: TrainingAdapterKind = CORE,
|
||||
base_model: str = MODEL,
|
||||
adapter_id: str = "adapter-1",
|
||||
weights: bool = True,
|
||||
) -> Path:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
manifest = _manifest(kind, base_model, adapter_id)
|
||||
(root / "adapter_manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
if weights:
|
||||
(root / "adapter_model.safetensors").write_bytes(b"stub weights")
|
||||
return root
|
||||
|
||||
|
||||
def _settings(tmp_path: Path) -> Settings:
|
||||
settings = Settings(home=tmp_path / "home", model_id=MODEL)
|
||||
settings.ensure_dirs()
|
||||
return settings
|
||||
|
||||
|
||||
def _cartridge(settings: Settings, database: Database, cartridge_id: str) -> Path:
|
||||
root = settings.home / "cartridges" / cartridge_id
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
now = datetime.now(UTC)
|
||||
manifest = CartridgeManifest(
|
||||
cartridge_id=cartridge_id,
|
||||
name="home",
|
||||
created_at=now,
|
||||
model_base=settings.model_id,
|
||||
source_count=0,
|
||||
atom_count=0,
|
||||
token_equivalent=0,
|
||||
artifacts={},
|
||||
dna=None,
|
||||
)
|
||||
database.write_cartridge(cartridge_id, "home", root, manifest.model_dump_json(), None, 0, now)
|
||||
set_mounts(database, [cartridge_id], now)
|
||||
return root
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
def __init__(self) -> None:
|
||||
self.evalled = False
|
||||
|
||||
def eval(self) -> None:
|
||||
self.evalled = True
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_stale_loads():
|
||||
adapters.clear_loads()
|
||||
yield
|
||||
adapters.clear_loads()
|
||||
|
||||
|
||||
def test_a_library_adapter_is_selected_for_its_kind(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
chosen = _write_adapter(settings.adapters_dir / "core")
|
||||
source, state, reason = adapters.resolve_library_adapter(settings)
|
||||
assert source is not None and source.path == chosen
|
||||
assert state == "available"
|
||||
assert reason == "selected from the adapter library"
|
||||
|
||||
|
||||
def test_an_adapter_on_another_base_model_is_deferred_naming_both_models(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
_write_adapter(settings.adapters_dir / "router", kind=TrainingAdapterKind.hme_router, base_model="Qwen/Qwen3.5-0.8B")
|
||||
source, state, reason = adapters.resolve_library_adapter(settings, TrainingAdapterKind.hme_router)
|
||||
assert source is None
|
||||
assert state == "deferred"
|
||||
assert "trained on Qwen/Qwen3.5-0.8B" in reason and f"runtime holds {MODEL}" in reason
|
||||
|
||||
|
||||
def test_a_manifest_without_weights_is_deferred_naming_the_missing_file(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
_write_adapter(settings.adapters_dir / "core", weights=False)
|
||||
source, state, reason = adapters.resolve_library_adapter(settings)
|
||||
assert source is None and state == "deferred"
|
||||
assert "adapter_model.safetensors" in reason
|
||||
|
||||
|
||||
def test_the_mounted_cartridge_copy_wins_over_the_library(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
database = Database(settings)
|
||||
_write_adapter(settings.adapters_dir / "core", adapter_id="library-copy")
|
||||
cartridge_root = _cartridge(settings, database, "cart-home")
|
||||
cartridge_copy = _write_adapter(cartridge_root / "adapters" / "hme_core_lora", adapter_id="cartridge-copy")
|
||||
sources, mounted = adapters.discover(settings, database)
|
||||
assert mounted == ["cart-home"]
|
||||
source, state, reason = adapters.resolve(sources, settings, CORE)
|
||||
assert source is not None and source.path == cartridge_copy
|
||||
assert state == "available" and "mounted cartridge cart-home" in reason
|
||||
report = adapters.plan(settings, database)
|
||||
row = next(item for item in report.adapters if item.kind == CORE)
|
||||
assert row.adapter_id == "cartridge-copy" and row.cartridge_id == "cart-home"
|
||||
|
||||
|
||||
def test_the_endpoint_lists_every_kind_with_a_reason(tmp_path: Path, monkeypatch) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
_write_adapter(settings.adapters_dir / "core")
|
||||
monkeypatch.setattr(api, "get_settings", lambda: settings)
|
||||
client = TestClient(api.create_app())
|
||||
body = client.get("/api/runtime/adapters").json()
|
||||
assert body["model_id"] == MODEL and body["mounted"] == []
|
||||
by_kind = {row["kind"]: row for row in body["adapters"]}
|
||||
assert set(by_kind) == {kind.value for kind in TrainingAdapterKind}
|
||||
assert by_kind[CORE.value]["state"] == "available"
|
||||
for kind in TrainingAdapterKind:
|
||||
if kind == CORE:
|
||||
continue
|
||||
assert by_kind[kind.value]["state"] == "missing"
|
||||
assert by_kind[kind.value]["reason"], f"{kind.value} carries no reason"
|
||||
|
||||
|
||||
def test_stub_weights_prove_the_load_branch_runs(tmp_path: Path, monkeypatch) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
chosen = _write_adapter(settings.adapters_dir / "core")
|
||||
attached: list[tuple[Any, Path]] = []
|
||||
|
||||
def fake_attach(model: Any, path: Path) -> _FakeModel:
|
||||
attached.append((model, path))
|
||||
return _FakeModel()
|
||||
|
||||
monkeypatch.setattr(adapters, "load_into_model", fake_attach)
|
||||
monkeypatch.setattr(QwenRuntime, "_load_base_model", lambda self: (_FakeModel(), object()))
|
||||
runtime = QwenRuntime(settings)
|
||||
model, _processor = runtime._load_transformers()
|
||||
assert attached == [(mock.ANY, chosen)], "peft was never asked to attach the stub adapter"
|
||||
assert model.evalled is True
|
||||
assert [record.adapter_id for record in adapters.load_records()] == ["adapter-1"]
|
||||
row = next(item for item in adapters.plan(settings, Database(settings)).adapters if item.kind == CORE)
|
||||
assert row.state == "loaded" and "attached to the live model" in row.reason
|
||||
|
||||
|
||||
def test_unload_clears_the_loaded_state(tmp_path: Path, monkeypatch) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
_write_adapter(settings.adapters_dir / "core")
|
||||
monkeypatch.setattr(adapters, "load_into_model", lambda model, path: _FakeModel())
|
||||
monkeypatch.setattr(QwenRuntime, "_load_base_model", lambda self: (_FakeModel(), object()))
|
||||
runtime = QwenRuntime(settings)
|
||||
runtime._load_transformers()
|
||||
assert adapters.load_records(), "nothing was recorded as loaded"
|
||||
runtime.unload()
|
||||
assert adapters.load_records() == []
|
||||
row = next(item for item in adapters.plan(settings, Database(settings)).adapters if item.kind == CORE)
|
||||
assert row.state == "available"
|
||||
|
||||
|
||||
def test_the_loser_of_the_choice_is_reported_as_shadowed(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
_write_adapter(settings.adapters_dir / "first", adapter_id="written_first")
|
||||
_write_adapter(settings.adapters_dir / "second", adapter_id="written_second")
|
||||
report = adapters.plan(settings, Database(settings))
|
||||
losers = [row for row in report.adapters if row.shadowed]
|
||||
assert [row.adapter_id for row in losers] == ["written_first"]
|
||||
assert "another adapter of this kind wins" in losers[0].reason
|
||||
winner = next(row for row in report.adapters if row.adapter_id == "written_second")
|
||||
assert winner.shadowed is False and winner.state == "available"
|
||||
|
||||
|
||||
def test_an_unreadable_manifest_names_the_file(tmp_path: Path) -> None:
|
||||
settings = _settings(tmp_path)
|
||||
broken = settings.adapters_dir / "broken"
|
||||
broken.mkdir(parents=True)
|
||||
(broken / "adapter_manifest.json").write_text("{not json", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="the adapter manifest is not a valid AdapterManifest"):
|
||||
adapters.library_sources(settings)
|
||||
|
||||
|
||||
def test_a_pinned_adapter_dir_without_a_manifest_says_the_path_is_missing(tmp_path: Path) -> None:
|
||||
settings = Settings(home=tmp_path / "home", model_id=MODEL, adapter_dir=tmp_path / "empty")
|
||||
settings.ensure_dirs()
|
||||
assert adapters.library_sources(settings) == []
|
||||
available, detail = QwenRuntime(settings)._transformers_status()
|
||||
assert available is False
|
||||
assert detail == f"Configured adapter path is missing: {tmp_path / 'empty'}"
|
||||
Reference in new issue
Block a user