diff --git a/CHECKLIST.md b/CHECKLIST.md index 23d0734..29e4ba4 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -228,11 +228,11 @@ - 修掉的真 bug：`CartridgeService.attach_adapter`（`cartridge.py:230`）把 adapter 拷进 `<卡带>/adapters/{kind}/`，但解析从来不看卡带目录，挂上去的东西运行时根本不加载。现在挂载卡带里的副本优先于库里的同名 adapter。 - 测试写出来才发现的第二个 bug：`ready.sort(key=_preference)` 丢了原来的 `reverse=True`，库里的 adapter 反而赢了挂载副本。 - 状态只由事实支撑：loaded（模型还活着——`record_load` 存 `weakref`，且记录挂在具体 `QwenRuntime._attached` 上，不是进程全局）/ available（manifest 与 `adapter_model.safetensors` 都在；模型没了就说「attached earlier in this process … that model is gone」）/ deferred（base_model 不匹配或缺权重文件，原因里写两个模型名）/ missing（磁盘上没有该 kind）。落选者单独一行 `shadowed=True`，且**只在真有赢家时**才有这行，原因点名赢家。 - - 排序键是三个：挂载卡带 > manifest 带 `cartridge_id` > 权重 mtime。第一个版本漏了中间那个，review 用 `/tmp/p13_pref.py` 量出 parent `9f0f799` 与 HEAD 选中的文件不同——`training.py:106` 写库内 adapter 时会带卡带标记，所以这不是风格问题。 - - 验收命令：`pytest backend/tests/test_adapters.py -q` → 13 passed。`test_stub_weights_prove_the_load_branch_runs` 用桩权重断言 `attached == [(ANY, 路径)]`、`model.evalled is True`、LoadRecord 存在、端点该行 `state == "loaded"`；`test_generation_reports_the_adapter_it_attached` 走完 transformers 生成路径，断言答案里的 adapter 路径 / kind / id / sha256 与 `output_tokens`；`test_unload_stops_claiming_a_model_that_is_gone` 断言 unload 后回到 available 并说明模型已消失，同时断言另一个实例不会继承这个声明。 + - 排序键是三个：挂载卡带 > manifest 带 `cartridge_id` > 权重 mtime。第一个版本漏了中间那个，reviewer 在 /tmp 里对比 parent `9f0f799` 与本提交选中的文件，量出两者不同（可复现做法：先写一个带 `cartridge_id` 的 adapter，再写一个不带但 mtime 更新的，看 `resolve` 选哪个）——`training.py:106` 写库内 adapter 时会带卡带标记，所以这不是风格问题。 + - 验收命令：`pytest backend/tests/test_adapters.py -q` → 15 passed。`test_stub_weights_prove_the_load_branch_runs` 用桩权重断言 `attached == [(ANY, 路径)]`、`model.evalled is True`、LoadRecord 存在、端点该行 `state == "loaded"`；`test_generation_reports_the_adapter_it_attached` 走完 transformers 生成路径，断言答案里的 adapter 路径 / kind / id / sha256 与 `output_tokens`；`test_unload_stops_claiming_a_model_that_is_gone` 断言 unload 后回到 available 并说明模型已消失，同时断言另一个实例不会继承这个声明；`test_a_loaded_row_names_the_adapter_that_is_actually_attached` 与 `test_one_unload_does_not_declare_another_live_model_gone` 钉住第二轮那两条（行里的身份列必须跟加载记录一致；一个模型 unload 不能说成另一个活模型消失了）。两条都做了变异检验：把行身份改回按解析结果取 → 前者 FAILED；让 `live_attachments()` 忽略存活 → 后两个 FAILED。 - 证明与生成解析一致：`proofs/checks_runtime.py` 的 `_generation_adapter()` 用 `discover` + `resolve`（含挂载卡带），不再用只看库的解析，否则同一份证明里 `resolved_adapter` 与 `adapter` 两块会指向不同副本还静默。 - 新文件 `backend/src/ely_eye/runtime_schemas.py`（30 行）：`schemas.py` 已 498 行，离 500 行红线 2 行，报告类型不放进去。 - - 门禁：`pytest -q` 225 passed；ruff 干净；mypy 冷缓存查 7 个改动文件 → 2 errors，与 HEAD 的 `runtime.py` 逐条相同（`qwen_vl_utils` 无 stubs、`model_class` 重赋值），净新增 0；pyright 同 7 文件 → 2 errors，HEAD 也是 2 条，都是 `process_vision_info` 的双返回（`vision_process.py:532-534` 由 `return_video_kwargs` 决定返回 2 或 3 元组，产品用默认值取 2 个，运行时正确）。 + - 门禁：`pytest -q` 227 passed；ruff 干净；mypy 冷缓存查 3 个改动源文件 → 2 errors，与 `git stash` 后同一命令在 HEAD 树上的实测完全同两条（`qwen_vl_utils` 无 stubs、`model_class` 重赋值），净新增 0；pyright 同文件 → 2 errors，HEAD 也是 2 条，都是 `process_vision_info` 的双返回（`vision_process.py:532-534` 由 `return_video_kwargs` 决定返回 2 或 3 元组，产品用默认值取 2 个，运行时正确）。 - 独立 review（同一个 `p1review` 里的 pi，模型 `strata/qwen3.8-flash-next-abliterated-iq3_s:high`）第一轮 **FAIL 6 条**，全部核实并改掉： 1. `plan()` 用 `chosen_paths` 判落选，把「被拒绝的候选」也当成落选者，于是没有任何 adapter 胜出时却说「another adapter of this kind wins」，还把真正的 deferred 原因盖掉了（reviewer 挂载 cart-home 复现，`hme_router`、`hme_vision_lora` 两行都在撒谎）。 2. `_LOADED` 是进程全局，而原因写「attached to the live model」——`api.py` 每个请求新建一个 `QwenRuntime`，`_model is None` 也报 loaded。改成 weakref + 实例字段。 @@ -241,6 +241,7 @@ 5. `_preference` 丢了 `manifest.cartridge_id is not None` 这个次级键，换掉了原来会加载哪个 adapter（见上）。 6. 证明检查与生成解析不一致（见上）。 reviewer 还指出 4 条生成路径与 `status()` 零覆盖，所以「没丢函数」当时只是读出来的——现在有了走完生成路径的测试。 + - 第二轮 **FAIL 1 条**（同一 reviewer，它自己建 worktree + `PYTHONPATH` 跑 harness，并记下 editable install 指向主树、不加 `PYTHONPATH` 的 worktree 测试会假绿）：`plan()` 里 loaded 行的 `reason` 按加载记录写，而 `path`/`adapter_id`/`base_model`/`origin`/`cartridge_id` 按当前解析结果写——实测：活模型挂 X，磁盘上又写了个更新的 Y，端点就返回 `adapter_id=adapter-Y` 却解释「attached from …/x-attached」。同一个根因的第二个症状：登记按 kind 存、后写覆盖，两个 runtime 时 unload 其中一个会把另一个还活着的模型说成「that model is gone」。修法：`LoadRecord` 直接携带 `AdapterSource`，行身份从记录取（`_choice()` 只有一个身份来源）；登记改成按 attachment 列表 + 各自 weakref，`loaded` 只看是否还有活模型持有，多于一个时原因里写明另一个。 - CLI 面：`ely-eye adapters` 打印同一份报告。 - [ ] **p1-4** 工程地基 - [ ] p1-4.1 pytest 从零到有（后端） diff --git a/backend/src/ely_eye/adapters.py b/backend/src/ely_eye/adapters.py index 047a438..ed56c6a 100644 --- a/backend/src/ely_eye/adapters.py +++ b/backend/src/ely_eye/adapters.py @@ -34,16 +34,25 @@ class AdapterSource: @dataclass(frozen=True) class LoadRecord: - """An adapter this process attached to a model.""" + """One adapter this process attached to one model.""" - kind: TrainingAdapterKind - adapter_id: str - path: str + source: AdapterSource sha256: str + @property + def kind(self) -> TrainingAdapterKind: + return self.source.manifest.kind -_LOADED: dict[str, LoadRecord] = {} -_LIVE: dict[str, weakref.ReferenceType[object]] = {} + @property + def adapter_id(self) -> str: + return self.source.manifest.adapter_id + + @property + def path(self) -> str: + return str(self.source.path) + + +_ATTACHMENTS: list[tuple[LoadRecord, weakref.ReferenceType[object]]] = [] def read_manifest(path: Path) -> AdapterManifest: @@ -153,31 +162,21 @@ def _preference(source: AdapterSource) -> tuple[bool, bool, float]: def load_records() -> list[LoadRecord]: - return list(_LOADED.values()) + """Every adapter still attached, or attached since the last load, in attachment order.""" + return [record for record, _reference in _ATTACHMENTS] -def live_kinds() -> set[str]: - """Kinds whose attached model is still alive, not merely attached once.""" - alive = {kind for kind, reference in _LIVE.items() if reference() is not None} - for kind in [kind for kind, reference in _LIVE.items() if reference() is None]: - del _LIVE[kind] - return alive - - -def clear_loads() -> None: - _LOADED.clear() - _LIVE.clear() +def live_attachments() -> list[LoadRecord]: + """Attachments whose model still exists; a dead model cannot be claimed as loaded.""" + return [record for record, reference in _ATTACHMENTS if reference() is not None] def record_load(source: AdapterSource, model: Any) -> LoadRecord: - record = LoadRecord( - kind=source.manifest.kind, - adapter_id=source.manifest.adapter_id, - path=str(source.path), - sha256=source.manifest.sha256, - ) - _LOADED[record.kind.value] = record - _LIVE[record.kind.value] = weakref.ref(model) + record = LoadRecord(source=source, sha256=source.manifest.sha256) + _ATTACHMENTS[:] = [ + (item, reference) for item, reference in _ATTACHMENTS if reference() is not None + ] + _ATTACHMENTS.append((record, weakref.ref(model))) logger.info("loaded adapter %s (%s) from %s", record.adapter_id, record.kind.value, record.path) return record @@ -197,44 +196,57 @@ def plan(settings: Settings, database: Database) -> AdapterLoadReport: sources, mounted = discover(settings, database) rows: list[AdapterChoice] = [] winners: dict[str, AdapterSource | None] = {} - alive = live_kinds() + attached = live_attachments() for kind in TrainingAdapterKind: source, state, reason = resolve(sources, settings, kind) winners[kind.value] = source - record = next((item for item in load_records() if item.kind == kind), None) - if record is not None: - if kind.value in alive: - state, reason = "loaded", f"attached to the live model from {record.path}" - else: - state = "available" - reason = f"attached earlier in this process from {record.path}; that model is gone" - rows.append( - AdapterChoice( - kind=kind, - state=state, - reason=reason, - path=str(source.path) if source else None, - adapter_id=source.manifest.adapter_id if source else None, - base_model=source.manifest.base_model if source else settings.model_id, - origin=source.origin if source else None, - cartridge_id=source.cartridge_id if source else None, - ) - ) + live = [record for record in attached if record.kind == kind] + earlier = next((record for record in reversed(load_records()) if record.kind == kind), None) + if live: + newest = live[-1] + state = "loaded" + reason = f"attached to the live model from {newest.path}" + held = sorted({record.adapter_id for record in live[:-1] if record.adapter_id != newest.adapter_id}) + if held: + reason = f"{reason}; another live model still holds {' and '.join(held)}" + rows.append(_choice(kind, state, reason, newest.source)) + continue + if earlier is not None and source is not None: + state = "available" + reason = f"{reason}; {earlier.adapter_id} was attached earlier in this process and that model is gone" + rows.append(_choice(kind, state, reason, source)) for source in sources: winner = winners.get(source.manifest.kind.value) if winner is None or source.path == winner.path: continue rows.append( - AdapterChoice( - kind=source.manifest.kind, - state="deferred", - reason=f"shadowed by {winner.manifest.adapter_id} for the current mount", - path=str(source.path), - adapter_id=source.manifest.adapter_id, - base_model=source.manifest.base_model, - origin=source.origin, - cartridge_id=source.cartridge_id, + _choice( + source.manifest.kind, + "deferred", + f"shadowed by {winner.manifest.adapter_id} for the current mount", + source, shadowed=True, ) ) return AdapterLoadReport(model_id=settings.model_id, mounted=mounted, adapters=rows) + + +def _choice( + kind: TrainingAdapterKind, + state: AdapterState, + reason: str, + source: AdapterSource | None, + shadowed: bool = False, +) -> AdapterChoice: + """Build one report row from the adapter the row is actually talking about.""" + return AdapterChoice( + kind=kind, + state=state, + reason=reason, + path=str(source.path) if source else None, + adapter_id=source.manifest.adapter_id if source else None, + base_model=source.manifest.base_model if source else None, + origin=source.origin if source else None, + cartridge_id=source.cartridge_id if source else None, + shadowed=shadowed, + ) diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index 40b9e85..a11b812 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -117,9 +117,9 @@ class _FakeProcessor: @pytest.fixture(autouse=True) def _no_stale_loads(): - adapters.clear_loads() + adapters._ATTACHMENTS.clear() yield - adapters.clear_loads() + adapters._ATTACHMENTS.clear() def test_a_library_adapter_is_selected_for_its_kind(tmp_path: Path) -> None: @@ -219,6 +219,43 @@ def test_unload_stops_claiming_a_model_that_is_gone(tmp_path: Path, monkeypatch) assert "that model is gone" in row.reason, row.reason +def test_a_loaded_row_names_the_adapter_that_is_actually_attached(tmp_path: Path, monkeypatch) -> None: + settings = _settings(tmp_path) + attached = _write_adapter(settings.adapters_dir / "x", adapter_id="adapter-x") + monkeypatch.setattr(adapters, "load_into_model", lambda model, path: _FakeModel()) + monkeypatch.setattr(QwenRuntime, "_load_base_model", lambda self: (_FakeModel(), object())) + runtime = QwenRuntime(settings) + runtime._load_transformers() + _write_adapter(settings.adapters_dir / "y", adapter_id="adapter-y") + row = next(item for item in adapters.plan(settings, Database(settings)).adapters if item.kind == CORE) + assert row.state == "loaded" + assert row.adapter_id == "adapter-x", f"the row promises {row.adapter_id} but {attached} is attached" + assert row.path == str(attached) and row.reason.endswith(str(attached)), row.reason + + +def test_one_unload_does_not_declare_another_live_model_gone(tmp_path: Path, monkeypatch) -> None: + settings = _settings(tmp_path) + _write_adapter(settings.adapters_dir / "x", adapter_id="adapter-x") + monkeypatch.setattr(adapters, "load_into_model", lambda model, path: _FakeModel()) + monkeypatch.setattr(QwenRuntime, "_load_base_model", lambda self: (_FakeModel(), object())) + first = QwenRuntime(settings) + first._load_transformers() + _write_adapter(settings.adapters_dir / "y", adapter_id="adapter-y") + second = QwenRuntime(settings) + second._load_transformers() + database = Database(settings) + row = next(item for item in adapters.plan(settings, database).adapters if item.kind == CORE) + assert row.state == "loaded" and row.adapter_id == "adapter-y" + assert "another live model still holds adapter-x" in row.reason, row.reason + first.unload() + row = next(item for item in adapters.plan(settings, database).adapters if item.kind == CORE) + assert row.state == "loaded" and row.adapter_id == "adapter-y" + assert "gone" not in row.reason, row.reason + second.unload() + row = next(item for item in adapters.plan(settings, database).adapters if item.kind == CORE) + assert row.state == "available" and "that model is gone" in row.reason, row.reason + + def test_a_rejected_candidate_does_not_claim_another_adapter_won(tmp_path: Path) -> None: settings = _settings(tmp_path) _write_adapter(settings.adapters_dir / "router", kind=TrainingAdapterKind.hme_router, base_model="Qwen/Qwen3.5-0.8B")