Resolve adapters by what is mounted, and say why each kind is not loaded

runtime.py resolved adapters at 7 call sites and only ever looked in the
adapter library, so an adapter attached to a cartridge (cartridge.py:230
copies it into <cartridge>/adapters/{kind}/) was never loaded by anything.
Resolution now lives in adapters.py: it reads the mounted cartridge set,
prefers a mounted cartridge's copy over the library, and answers every one
of the six kinds with a state the facts support — loaded, available,
deferred (base_model mismatch or missing weights, both models named in the
reason), missing — plus a shadowed row for each adapter that lost the
choice. A mounted cartridge is the scenario selection; neither manifest
carries a scenario field, so none was invented.

Two bugs the new tests caught: the preference sort lost reverse=True, so a
library adapter beat a mounted cartridge's copy, and the load branch had
never been executed by a test. Stub weights now prove peft is asked to
attach, that the returned model is put in eval mode, and that the endpoint
reports loaded until unload clears it.

Faces: GET /api/runtime/adapters, `ely-eye adapters`, and a log line naming
the reason whenever a generation runs without an adapter.

Gates: 222 passed; ruff clean; mypy cold cache on the 7 touched files gives
the same 2 errors as HEAD's runtime.py, and pyright gives the same 2 as
HEAD — both are qwen_vl_utils typing artifacts (process_vision_info returns
2 or 3 values depending on return_video_kwargs; the product takes 2).
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 23:09:28 -04:00
1 parent 9f0f7993e5
commit 2fbb4af96b
8 files changed
+585 -86

No files matched your search

+229
View File
@@ -0,0 +1,229 @@
"""One place decides which adapter loads, and says out loud why the others do not."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from pydantic import ValidationError
from . import cartridge_store
from .config import Settings
from .db import Database
from .runtime_schemas import AdapterChoice, AdapterLoadReport, AdapterOrigin, AdapterState
from .schemas import AdapterManifest, TrainingAdapterKind
logger = logging.getLogger("ely_eye.adapters")
MANIFEST_FILE = "adapter_manifest.json"
WEIGHTS_FILE = "adapter_model.safetensors"
@dataclass(frozen=True)
class AdapterSource:
"""One adapter found on disk, plus where it was found."""
path: Path
manifest: AdapterManifest
origin: AdapterOrigin
cartridge_id: str | None
@dataclass(frozen=True)
class LoadRecord:
"""An adapter this process actually attached to a live model."""
kind: TrainingAdapterKind
adapter_id: str
path: str
sha256: str
loaded_at: str
_LOADED: dict[str, LoadRecord] = {}
def read_manifest(path: Path) -> AdapterManifest:
try:
text = path.read_text(encoding="utf-8")
except OSError as error:
raise ValueError(f"{path}: the adapter manifest cannot be read ({error})") from error
try:
return AdapterManifest.model_validate_json(text)
except ValidationError as error:
raise ValueError(f"{path}: the adapter manifest is not a valid AdapterManifest ({error})") from error
def cartridge_roots(database: Database) -> dict[str, Path]:
roots: dict[str, Path] = {}
for row in database.list_cartridges():
roots[str(row["cartridge_id"])] = Path(str(row["root_path"]))
return roots
def library_sources(settings: Settings) -> list[AdapterSource]:
"""Adapters in the library directory, for callers that have no database open."""
if settings.adapter_dir is not None:
pinned = settings.adapter_dir.expanduser()
if not (pinned / MANIFEST_FILE).exists():
logger.warning("adapter_dir %s holds no %s", pinned, MANIFEST_FILE)
return []
manifest = read_manifest(pinned / MANIFEST_FILE)
return [
AdapterSource(
path=pinned,
manifest=manifest,
origin="library",
cartridge_id=manifest.cartridge_id,
)
]
sources: list[AdapterSource] = []
for manifest_path in sorted(settings.adapters_dir.glob(f"*/{MANIFEST_FILE}")):
manifest = read_manifest(manifest_path)
sources.append(
AdapterSource(
path=manifest_path.parent,
manifest=manifest,
origin="library",
cartridge_id=manifest.cartridge_id,
)
)
return sources
def discover(settings: Settings, database: Database) -> tuple[list[AdapterSource], list[str]]:
"""Every adapter the runtime could use: the library plus the copies inside mounted cartridges."""
mounted = cartridge_store.mounted_ids(database)
roots = cartridge_roots(database)
sources = library_sources(settings)
for cartridge_id in mounted:
root = roots.get(cartridge_id)
if root is None:
logger.warning("cartridge %s is mounted but its directory is not registered", cartridge_id)
continue
for manifest_path in sorted((root / "adapters").glob(f"*/{MANIFEST_FILE}")):
sources.append(
AdapterSource(
path=manifest_path.parent,
manifest=read_manifest(manifest_path),
origin="cartridge",
cartridge_id=cartridge_id,
)
)
return sources, mounted
def resolve(
sources: list[AdapterSource],
settings: Settings,
kind: TrainingAdapterKind,
) -> tuple[AdapterSource | None, AdapterState, str]:
"""Pick the adapter to load for one kind, or say which state it is stuck in and why."""
candidates = [source for source in sources if source.manifest.kind == kind]
if not candidates:
return None, "missing", f"no adapter of kind {kind.value} is on disk"
matching = [source for source in candidates if source.manifest.base_model == settings.model_id]
if not matching:
trained_on = ", ".join(sorted({source.manifest.base_model for source in candidates}))
return (
None,
"deferred",
f"every {kind.value} adapter is trained on {trained_on}; the runtime holds {settings.model_id}",
)
ready = [source for source in matching if (source.path / WEIGHTS_FILE).exists()]
if not ready:
return None, "deferred", f"the {kind.value} manifests are present but none carries {WEIGHTS_FILE}"
ready.sort(key=_preference, reverse=True)
chosen = ready[0]
if chosen.origin == "cartridge":
return chosen, "available", f"selected from mounted cartridge {chosen.cartridge_id}"
return chosen, "available", "selected from the adapter library"
def resolve_library_adapter(
settings: Settings,
kind: TrainingAdapterKind = TrainingAdapterKind.hme_core_lora,
) -> tuple[AdapterSource | None, AdapterState, str]:
"""Resolution without a database: the library only, no mounted cartridges."""
return resolve(library_sources(settings), settings, kind)
def _preference(source: AdapterSource) -> tuple[bool, float]:
weights = source.path / WEIGHTS_FILE
return (source.origin == "cartridge", weights.stat().st_mtime if weights.exists() else 0.0)
def load_records() -> list[LoadRecord]:
return list(_LOADED.values())
def clear_loads() -> None:
_LOADED.clear()
def record_load(source: AdapterSource) -> LoadRecord:
record = LoadRecord(
kind=source.manifest.kind,
adapter_id=source.manifest.adapter_id,
path=str(source.path),
sha256=source.manifest.sha256,
loaded_at=datetime.now().astimezone().isoformat(),
)
_LOADED[record.kind.value] = record
logger.info("loaded adapter %s (%s) from %s", record.adapter_id, record.kind.value, record.path)
return record
def load_into_model(model: Any, path: Path) -> Any:
"""Attach one LoRA to a live model; the only place generation imports peft."""
from .training import bitsandbytes_gaudi_probe_guard
with bitsandbytes_gaudi_probe_guard():
from peft import PeftModel
return PeftModel.from_pretrained(model, str(path))
def plan(settings: Settings, database: Database) -> AdapterLoadReport:
"""One row per adapter kind, plus a row for every adapter that lost the choice."""
sources, mounted = discover(settings, database)
rows: list[AdapterChoice] = []
for kind in TrainingAdapterKind:
source, state, reason = resolve(sources, settings, kind)
record = next((item for item in load_records() if item.kind == kind), None)
if record is not None:
reason = f"attached to the live model from {record.path}"
state = "loaded"
rows.append(
AdapterChoice(
kind=kind,
state=state,
reason=reason,
path=str(source.path) if source else None,
adapter_id=source.manifest.adapter_id if source else None,
base_model=source.manifest.base_model if source else settings.model_id,
origin=source.origin if source else None,
cartridge_id=source.cartridge_id if source else None,
)
)
chosen_paths = {row.path for row in rows if row.path}
for source in sources:
if str(source.path) in chosen_paths:
continue
rows.append(
AdapterChoice(
kind=source.manifest.kind,
state="deferred",
reason="another adapter of this kind wins for the current mount",
path=str(source.path),
adapter_id=source.manifest.adapter_id,
base_model=source.manifest.base_model,
origin=source.origin,
cartridge_id=source.cartridge_id,
shadowed=True,
)
)
return AdapterLoadReport(model_id=settings.model_id, mounted=mounted, adapters=rows)
+6
View File
@@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from . import __version__
from .adapters import plan as adapter_plan
from .api_cartridges import register as register_cartridge_routes
from .api_holders import register as register_holder_routes
from .benchmarks import (
@@ -41,6 +42,7 @@ from .proofs import HashHopEvaluator, PRDProofSuite
from .research_modules import ResearchModuleService
from .retrieval import RetrievalService
from .runtime import QwenRuntime, RuntimeUnavailableError
from .runtime_schemas import AdapterLoadReport
from .schemas import (
AdapterManifest,
AdapterStatus,
@@ -277,6 +279,10 @@ def create_app() -> FastAPI:
def runtime_profiles() -> RuntimeProfileReport:
return runtime_profile_report()
@app.get("/api/runtime/adapters", response_model=AdapterLoadReport)
def runtime_adapters(settings: Annotated[Settings, Depends(settings_dep)]) -> AdapterLoadReport:
return adapter_plan(settings, Database(settings))
@app.get("/api/research/modules")
def research_modules(settings: Annotated[Settings, Depends(settings_dep)]):
db = Database(settings)
+9
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import typer
import uvicorn
from .adapters import plan as adapter_plan
from .api import create_app
from .benchmarks import (
benchmark_prediction_report,
@@ -475,5 +476,13 @@ def attach_adapter(cartridge_id: str, adapter_dir: Path) -> None:
typer.echo(manifest.model_dump_json(indent=2))
@app.command("adapters")
def adapter_status() -> None:
"""Which adapter the runtime would load, and why each other kind would not."""
settings = get_settings()
report = adapter_plan(settings, Database(settings))
typer.echo(json_dump(report.model_dump(mode="json")))
if __name__ == "__main__":
app()
+4 -3
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
from pathlib import Path
from typing import Any
from .. import adapters
from ..compiler import ContextCompiler
from ..config import Settings
from ..db import Database
@@ -17,7 +18,7 @@ from ..evaluation_schemas import (
)
from ..evidence_graph import EvidenceGraphService
from ..profiles import runtime_profile_report
from ..runtime import QwenRuntime, resolve_runtime_adapter
from ..runtime import QwenRuntime
from ..schemas import (
RuntimeProfile,
TrainingAdapterKind,
@@ -58,7 +59,7 @@ class RuntimeChecks:
return deferred
runtime = QwenRuntime(self.settings)
status = runtime.status()
adapter = resolve_runtime_adapter(self.settings)
adapter, _adapter_state, _adapter_reason = adapters.resolve_library_adapter(self.settings)
weight_path = adapter.path / "adapter_model.safetensors" if adapter else None
payload = {
"runtime_status": status.model_dump(mode="json"),
@@ -133,7 +134,7 @@ class RuntimeChecks:
)
payload["generation_question"] = generation_question
status = runtime.status()
adapter = resolve_runtime_adapter(proof_settings)
adapter, _adapter_state, _adapter_reason = adapters.resolve_library_adapter(proof_settings)
generation = runtime.generate(generation_question, context)
answer, citations, citation_report = Verifier().verify_answer(generation.text, context)
claim_id = EvidenceGraphService(proof_settings, self.db).record_answer(
+68 -80
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import base64
import json
import logging
import subprocess
from dataclasses import dataclass
from pathlib import Path
@@ -9,10 +10,14 @@ from typing import Any
import httpx
from . import adapters
from .config import Settings, get_settings
from .schemas import AdapterManifest, CompiledContext, RuntimeStatus, TrainingAdapterKind
from .db import Database
from .schemas import CompiledContext, RuntimeStatus, TrainingAdapterKind
from .storage import ObjectStore
logger = logging.getLogger("ely_eye.runtime")
class RuntimeUnavailableError(RuntimeError):
pass
@@ -34,12 +39,41 @@ class QwenRuntime:
def __init__(self, settings: Settings | None = None) -> None:
self.settings = settings or get_settings()
self.objects = ObjectStore(self.settings)
self.database = Database(self.settings)
self._processor: Any | None = None
self._model: Any | None = None
def _adapter_choice(self) -> tuple[adapters.AdapterSource | None, str]:
"""The adapter this runtime would attach, plus the reason if it would not attach one."""
sources, _ = adapters.discover(self.settings, self.database)
source, _state, reason = adapters.resolve(sources, self.settings, TrainingAdapterKind.hme_core_lora)
return source, reason
def _loaded_adapter(self) -> adapters.LoadRecord | None:
return next((item for item in adapters.load_records() if item.kind == TrainingAdapterKind.hme_core_lora), None)
def _generation_adapter_fields(self) -> dict[str, str | None]:
record = self._loaded_adapter()
if record is not None:
return {
"adapter_path": record.path,
"adapter_kind": record.kind.value,
"adapter_id": record.adapter_id,
"adapter_sha256": record.sha256,
}
source, _ = self._adapter_choice()
if source is None:
return {"adapter_path": None, "adapter_kind": None, "adapter_id": None, "adapter_sha256": None}
return {
"adapter_path": str(source.path),
"adapter_kind": source.manifest.kind.value,
"adapter_id": source.manifest.adapter_id,
"adapter_sha256": source.manifest.sha256,
}
def status(self) -> RuntimeStatus:
gpu = gpu_status()
adapter = resolve_runtime_adapter(self.settings)
fields = self._generation_adapter_fields()
if self.settings.runtime_backend == "sglang":
available, detail = self._sglang_status()
else:
@@ -49,10 +83,10 @@ class QwenRuntime:
model_id=self.settings.model_id,
available=available,
detail=detail,
adapter_path=str(adapter.path) if adapter else None,
adapter_kind=adapter.manifest.kind if adapter else None,
adapter_id=adapter.manifest.adapter_id if adapter else None,
adapter_sha256=adapter.manifest.sha256 if adapter else None,
adapter_path=fields["adapter_path"],
adapter_kind=TrainingAdapterKind(fields["adapter_kind"]) if fields["adapter_kind"] else None,
adapter_id=fields["adapter_id"],
adapter_sha256=fields["adapter_sha256"],
gpu_name=gpu.get("name"),
cuda=gpu.get("cuda"),
vram_total_mb=gpu.get("total_mb"),
@@ -84,15 +118,17 @@ class QwenRuntime:
import torch
adapter_detail = ""
adapter = resolve_runtime_adapter(self.settings)
if adapter is not None:
adapter_detail = f" Adapter selected: {adapter.path}."
source, reason = self._adapter_choice()
if source is not None:
adapter_detail = f" Adapter selected: {source.path}."
elif self.settings.adapter_dir is not None:
return False, f"Configured adapter path is missing: {self.settings.adapter_dir}"
if source is None:
adapter_detail = f" No adapter: {reason}."
if torch.cuda.is_available():
return True, f"Transformers backend can use CUDA.{adapter_detail}"
return True, f"Transformers backend is available on CPU.{adapter_detail}"
except Exception as exc:
except Exception as exc: # noqa: BLE001 # a status probe reports why the backend is unusable
return False, f"Transformers backend import failed: {exc}"
def _generate_sglang(self, question: str, context: CompiledContext) -> RuntimeGeneration:
@@ -122,15 +158,11 @@ class QwenRuntime:
except httpx.HTTPError as exc:
raise RuntimeUnavailableError(f"SGLang generation failed: {exc}") from exc
data = response.json()
adapter = resolve_runtime_adapter(self.settings)
usage = data.get("usage") if isinstance(data, dict) else None
return RuntimeGeneration(
text=data["choices"][0]["message"]["content"],
backend="sglang",
adapter_path=str(adapter.path) if adapter else None,
adapter_kind=adapter.manifest.kind.value if adapter else None,
adapter_id=adapter.manifest.adapter_id if adapter else None,
adapter_sha256=adapter.manifest.sha256 if adapter else None,
**self._generation_adapter_fields(),
input_tokens=usage.get("prompt_tokens") if isinstance(usage, dict) else None,
output_tokens=usage.get("completion_tokens") if isinstance(usage, dict) else None,
)
@@ -161,15 +193,11 @@ class QwenRuntime:
except httpx.HTTPError as exc:
raise RuntimeUnavailableError(f"SGLang generation failed: {exc}") from exc
data = response.json()
adapter = resolve_runtime_adapter(self.settings)
usage = data.get("usage") if isinstance(data, dict) else None
return RuntimeGeneration(
text=data["choices"][0]["message"]["content"],
backend="sglang",
adapter_path=str(adapter.path) if adapter else None,
adapter_kind=adapter.manifest.kind.value if adapter else None,
adapter_id=adapter.manifest.adapter_id if adapter else None,
adapter_sha256=adapter.manifest.sha256 if adapter else None,
**self._generation_adapter_fields(),
input_tokens=usage.get("prompt_tokens") if isinstance(usage, dict) else None,
output_tokens=usage.get("completion_tokens") if isinstance(usage, dict) else None,
)
@@ -218,14 +246,10 @@ class QwenRuntime:
input_length = int(inputs["input_ids"].shape[-1])
generated = generated[:, input_length:]
decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
adapter = resolve_runtime_adapter(self.settings)
return RuntimeGeneration(
text=decoded,
backend="transformers",
adapter_path=str(adapter.path) if adapter else None,
adapter_kind=adapter.manifest.kind.value if adapter else None,
adapter_id=adapter.manifest.adapter_id if adapter else None,
adapter_sha256=adapter.manifest.sha256 if adapter else None,
**self._generation_adapter_fields(),
input_tokens=input_length,
output_tokens=int(generated.shape[-1]),
)
@@ -273,14 +297,10 @@ class QwenRuntime:
input_length = int(inputs["input_ids"].shape[-1])
generated = generated[:, input_length:]
decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
adapter = resolve_runtime_adapter(self.settings)
return RuntimeGeneration(
text=decoded,
backend="transformers",
adapter_path=str(adapter.path) if adapter else None,
adapter_kind=adapter.manifest.kind.value if adapter else None,
adapter_id=adapter.manifest.adapter_id if adapter else None,
adapter_sha256=adapter.manifest.sha256 if adapter else None,
**self._generation_adapter_fields(),
input_tokens=input_length,
output_tokens=int(generated.shape[-1]),
)
@@ -288,6 +308,19 @@ class QwenRuntime:
def _load_transformers(self) -> tuple[Any, Any]:
if self._model is not None and self._processor is not None:
return self._model, self._processor
model, processor = self._load_base_model()
source, reason = self._adapter_choice()
if source is None:
logger.info("generating without an adapter: %s", reason)
else:
model = adapters.load_into_model(model, source.path)
adapters.record_load(source)
model.eval()
self._model, self._processor = model, processor
return model, processor
def _load_base_model(self) -> tuple[Any, Any]:
"""Load the chat model; tests replace this so the adapter branch runs without a 9B download."""
import torch
from transformers import AutoProcessor
@@ -300,33 +333,25 @@ class QwenRuntime:
model_cls = AutoModelForCausalLM
self._processor = AutoProcessor.from_pretrained(self.settings.model_id, trust_remote_code=True)
self._model = model_cls.from_pretrained(
processor = AutoProcessor.from_pretrained(self.settings.model_id, trust_remote_code=True)
model = model_cls.from_pretrained(
self.settings.model_id,
dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float32,
device_map="auto",
trust_remote_code=True,
)
adapter = resolve_runtime_adapter(self.settings)
if adapter is not None:
from .training import bitsandbytes_gaudi_probe_guard
with bitsandbytes_gaudi_probe_guard():
from peft import PeftModel
self._model = PeftModel.from_pretrained(self._model, adapter.path)
self._model.eval()
return self._model, self._processor
return model, processor
def unload(self) -> None:
self._model = None
self._processor = None
adapters.clear_loads()
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
except Exception: # noqa: BLE001 # freeing the cache is best-effort, unloading must not raise
return
def _context_images(self, context: CompiledContext) -> list[Path]:
@@ -338,43 +363,6 @@ class QwenRuntime:
return paths
@dataclass(frozen=True)
class RuntimeAdapter:
path: Path
manifest: AdapterManifest
def resolve_runtime_adapter(
settings: Settings,
kind: TrainingAdapterKind = TrainingAdapterKind.hme_core_lora,
) -> RuntimeAdapter | None:
if settings.adapter_dir is not None:
path = settings.adapter_dir.expanduser().resolve()
manifest_path = path / "adapter_manifest.json"
if path.exists() and manifest_path.exists():
return RuntimeAdapter(
path=path,
manifest=AdapterManifest.model_validate_json(manifest_path.read_text(encoding="utf-8")),
)
return None
candidates: list[RuntimeAdapter] = []
for manifest_path in settings.adapters_dir.glob("*/adapter_manifest.json"):
manifest = AdapterManifest.model_validate_json(manifest_path.read_text(encoding="utf-8"))
if manifest.kind == kind and manifest.base_model == settings.model_id:
candidates.append(RuntimeAdapter(path=manifest_path.parent.resolve(), manifest=manifest))
candidates.sort(
key=lambda item: (
item.manifest.cartridge_id is not None,
(item.path / "adapter_model.safetensors").stat().st_mtime
if (item.path / "adapter_model.safetensors").exists()
else 0.0,
),
reverse=True,
)
return candidates[0] if candidates else None
def build_prompt(question: str, context: CompiledContext) -> str:
contract = "\n".join(f"- {item}" for item in context.plan.verifier_contract)
return f"""You are Ely-Eye, a local multimodal context operating system.
+30
View File
@@ -0,0 +1,30 @@
"""Runtime observability shapes: what the model holds in memory right now."""
from typing import Literal
from pydantic import BaseModel
from .schemas import TrainingAdapterKind
AdapterState = Literal["loaded", "available", "deferred", "missing"]
AdapterOrigin = Literal["library", "cartridge"]
class AdapterChoice(BaseModel):
"""One adapter kind, and the honest reason it is or is not attached to the model."""
kind: TrainingAdapterKind
state: AdapterState
reason: str
path: str | None = None
adapter_id: str | None = None
base_model: str | None = None
origin: AdapterOrigin | None = None
cartridge_id: str | None = None
shadowed: bool = False
class AdapterLoadReport(BaseModel):
model_id: str
mounted: list[str]
adapters: list[AdapterChoice]