Make a cartridge a scope you mount, not only a bundle you export

A cartridge used to be an export: materialize it, hash it, and never let it
change behaviour again. The transactive-memory promise needs the opposite —
mounting the home cartridge must make the work cartridge's memories
unfindable, and appending to a cartridge must make a new memory findable
through it.

Cartridge scope (schema v2 -> v3):
- cartridge_atoms (which atoms are in which cartridge) and cartridge_mounts
  (which cartridges this machine has mounted now); both rebuilt on reopen,
  because a mount is a machine-local state, not a fact about the data.
- mount / switch / unmount / append refuse a cartridge that was built for a
  different runtime, naming every differing field instead of a boolean.
- append rewrites atoms.parquet and sparse_index.bm25.zst, recomputes DNA and
  the KV fingerprint, and keeps the cartridges row in one SQLite transaction;
  the files are written first so a failed export cannot leave a half-cartridge
  in the database.
- RetrievalService.search() now ranks inside the mounted scope by default, and
  an explicit cartridge_ids filter overrides the mount. Nothing mounted means
  the whole library, so an empty store behaves as it did before.
- forget() removes the forgotten atoms from every cartridge's membership:
  forgetting must not leave a memory retrievable through a cartridge.

KV fingerprint (kv_fingerprint.py, stored in the manifest):
- model / tokenizer / chat-template hashes, the RoPE block (factor,
  original_max_position_embeddings, mrope_interleaved, mrope_section,
  partial_rotary_factor), the adapter hash, the context length, the engine
  version, and the fingerprint format version.
- A snapshot we cannot read is recorded as unverifiable and listed in
  unverifiable_fields; a blind machine refuses to mount rather than pretending
  the runtime matches, because a wrong RoPE factor silently corrupts retrieval.

Two defects found by running the new tests, not by reading:
- atoms.parquet could not export a memory at all: pyarrow refuses a struct with
  no child field, so metadata={} failed the whole table. The six JSON columns
  are now written as JSON text and decoded by the cartridge reader.
- materialize() inserted cartridge membership before the cartridges row existed,
  so the foreign key rejected every cartridge.

api.py crossed 500 lines, so the cartridge and mount routes moved to
api_cartridges.py. That module must not use `from __future__ import
annotations`: FastAPI then resolves Depends(settings_dep) against module
globals, where the injected dependency is only a parameter of register(), and
every route starts demanding settings as a query parameter.

Verification: 138 pytest passed (26 new: 10 fingerprint, 16 scope), Go
build/vet/gofmt/test clean, and scripts/mcp_write_smoke.sh now proves the whole
loop across two processes - mount narrows search, a memory written after the
upload is invisible until append_to_cartridge, unmount restores the library,
and rewriting the RoPE factor makes mount_cartridge refuse with "rope: factor:
cartridge 4.0 vs local 2.0". ruff clean on every touched file (the 20 remaining
package errors are in files this change did not touch; HEAD had 25). mypy: one
error, the pre-existing np.savez stub artifact at cartridge_assets.py:329
(HEAD:320). Repo pyright: 62 errors, unchanged from the p0-3 ledger.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 16:40:27 -04:00
1 parent 674c88e84e
commit 9fe13fffb3
21 files changed
+1492 -153

No files matched your search

+132 -90
View File
@@ -4,16 +4,34 @@ import hashlib
import json
import shutil
import sqlite3
from datetime import datetime, timezone
from datetime import UTC, datetime
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
import zstandard as zstd
from . import cartridge_store
from .config import Settings, get_settings
from .db import Database
from .schemas import AdapterManifest, CartridgeManifest, EvidenceAtom, SourceRecord
from .kv_fingerprint import compute_fingerprint, fingerprint_diff, unverifiable_fields
from .schemas import (
AdapterManifest,
CartridgeManifest,
EvidenceAtom,
MountState,
SourceRecord,
utc_now,
)
JSON_PARQUET_COLUMNS = (
"embedding_refs",
"layout",
"relations",
"trust",
"metadata",
"visibility",
)
class CartridgeService:
@@ -27,7 +45,7 @@ class CartridgeService:
atoms: list[EvidenceAtom],
sources: list[SourceRecord],
) -> CartridgeManifest:
created_at = datetime.now(timezone.utc)
created_at = datetime.now(UTC)
cartridge_id = self._cartridge_id(name, created_at)
root = self.settings.cartridge_dir / cartridge_id
eval_dir = root / "eval_proofs"
@@ -55,6 +73,7 @@ class CartridgeService:
source_count=len(sources),
atom_count=len(atoms),
token_equivalent=sum(atom.token_equivalent for atom in atoms),
kv_fingerprint=compute_fingerprint(self.settings, root / "adapters"),
artifacts={
"manifest": "manifest.json",
"atoms": "atoms.parquet",
@@ -64,20 +83,8 @@ class CartridgeService:
"eval_proofs": "eval_proofs/",
},
)
manifest_path = root / "manifest.json"
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
dna = self.compute_dna(root)
manifest.dna = dna
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
self.db.write_cartridge(
cartridge_id=cartridge_id,
name=name,
root_path=root,
manifest_json=manifest.model_dump_json(),
dna=dna,
token_equivalent=manifest.token_equivalent,
created_at=created_at,
)
self._commit(manifest, root)
cartridge_store.set_membership(self.db, cartridge_id, [atom.atom_id for atom in atoms])
return manifest
def compute_dna(self, root: Path) -> str:
@@ -96,18 +103,102 @@ class CartridgeService:
manifests.append(CartridgeManifest.model_validate_json(row["manifest_json"]))
return manifests
def mount_state(self) -> MountState:
mounted = cartridge_store.mounted_ids(self.db)
return MountState(
mounted=mounted,
scoped=bool(mounted),
atoms_in_scope=len(cartridge_store.atoms_of_cartridges(self.db, mounted)),
)
def mount(self, cartridge_id: str) -> CartridgeManifest:
"""Add one cartridge to the mounted set; a mounted search answers only from it."""
root, manifest = self._require(cartridge_id)
self._refuse_if_incompatible(manifest, root)
mounted = cartridge_store.mounted_ids(self.db)
if cartridge_id not in mounted:
cartridge_store.set_mounts(self.db, [*mounted, cartridge_id], utc_now())
return manifest
def switch(self, cartridge_id: str) -> CartridgeManifest:
"""Mount one cartridge and nothing else."""
root, manifest = self._require(cartridge_id)
self._refuse_if_incompatible(manifest, root)
cartridge_store.set_mounts(self.db, [cartridge_id], utc_now())
return manifest
def unmount(self, cartridge_id: str) -> MountState:
self._require(cartridge_id)
mounted = cartridge_store.mounted_ids(self.db)
if cartridge_id not in mounted:
raise ValueError(f"{cartridge_id} is not mounted; nothing to unmount")
cartridge_store.set_mounts(
self.db, [held for held in mounted if held != cartridge_id], utc_now()
)
return self.mount_state()
def append(self, cartridge_id: str, atom_ids: list[str]) -> CartridgeManifest:
"""Add memories to an existing cartridge; the bundle and its DNA follow the membership."""
self._require(cartridge_id)
if not atom_ids:
raise ValueError("append needs at least one atom")
atoms = self.db.get_atoms(atom_ids)
found = {atom.atom_id for atom in atoms}
missing = [atom_id for atom_id in atom_ids if atom_id not in found]
if missing:
raise ValueError(f"cannot append, these atoms are not in the store: {', '.join(missing)}")
cartridge_store.add_membership(self.db, cartridge_id, atom_ids)
held = self.db.get_atoms(sorted(cartridge_store.atoms_of_cartridge(self.db, cartridge_id)))
sources = self.db.get_sources(sorted({atom.source_id for atom in held}))
return self.rebuild(cartridge_id, held, sources)
def _require(self, cartridge_id: str) -> tuple[Path, CartridgeManifest]:
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
if not rows:
raise ValueError(f"Unknown cartridge: {cartridge_id}")
row = rows[0]
return Path(row["root_path"]), CartridgeManifest.model_validate_json(row["manifest_json"])
def _refuse_if_incompatible(self, manifest: CartridgeManifest, root: Path) -> None:
"""KV bytes are only valid under the settings that produced them."""
stored = manifest.kv_fingerprint
if stored is None:
raise ValueError(f"{manifest.cartridge_id} carries no kv fingerprint; rebuild it before mounting")
live = compute_fingerprint(self.settings, root / "adapters")
gaps = unverifiable_fields(live)
if gaps:
raise ValueError(
f"cannot verify {manifest.cartridge_id} on this machine: {', '.join(gaps)} are unknown; "
"set ELY_EYE_MODEL_DIR to the model snapshot"
)
differences = fingerprint_diff(stored, live)
if differences:
raise ValueError(
f"{manifest.cartridge_id} was built for a different runtime; refusing to mount: "
+ "; ".join(differences)
)
def _commit(self, manifest: CartridgeManifest, root: Path) -> None:
"""Re-hash the bundle, then record the manifest that describes those bytes."""
manifest.dna = self.compute_dna(root)
(root / "manifest.json").write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
self.db.write_cartridge(
cartridge_id=manifest.cartridge_id,
name=manifest.name,
root_path=root,
manifest_json=manifest.model_dump_json(),
dna=manifest.dna,
token_equivalent=manifest.token_equivalent,
created_at=manifest.created_at,
)
def rebuild(
self,
cartridge_id: str,
atoms: list[EvidenceAtom],
sources: list[SourceRecord],
) -> CartridgeManifest:
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
if not rows:
raise ValueError(f"Unknown cartridge: {cartridge_id}")
row = rows[0]
root = Path(row["root_path"])
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
root, manifest = self._require(cartridge_id)
root.mkdir(parents=True, exist_ok=True)
self._write_atoms_parquet(root / "atoms.parquet", atoms)
@@ -133,24 +224,11 @@ class CartridgeService:
)
manifest.dna = self.compute_dna(root)
(root / "manifest.json").write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
self.db.write_cartridge(
cartridge_id=manifest.cartridge_id,
name=manifest.name,
root_path=root,
manifest_json=manifest.model_dump_json(),
dna=manifest.dna,
token_equivalent=manifest.token_equivalent,
created_at=manifest.created_at,
)
self._commit(manifest, root)
return manifest
def attach_adapter(self, cartridge_id: str, adapter_dir: Path) -> CartridgeManifest:
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
if not rows:
raise ValueError(f"Unknown cartridge: {cartridge_id}")
row = rows[0]
root = Path(row["root_path"])
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
root, manifest = self._require(cartridge_id)
adapter_manifest_path = adapter_dir / "adapter_manifest.json"
if not adapter_manifest_path.exists():
raise ValueError(f"Missing adapter manifest: {adapter_manifest_path}")
@@ -163,18 +241,7 @@ class CartridgeService:
manifest.artifacts[f"adapter_{adapter_manifest.kind.value}"] = (
adapter_root.relative_to(root).as_posix() + "/"
)
manifest_path = root / "manifest.json"
manifest.dna = self.compute_dna(root)
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
self.db.write_cartridge(
cartridge_id=manifest.cartridge_id,
name=manifest.name,
root_path=root,
manifest_json=manifest.model_dump_json(),
dna=manifest.dna,
token_equivalent=manifest.token_equivalent,
created_at=manifest.created_at,
)
self._commit(manifest, root)
return manifest
def update_artifacts(
@@ -183,59 +250,34 @@ class CartridgeService:
artifacts: dict[str, str],
token_equivalent: int | None = None,
) -> CartridgeManifest:
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
if not rows:
raise ValueError(f"Unknown cartridge: {cartridge_id}")
row = rows[0]
root = Path(row["root_path"])
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
root, manifest = self._require(cartridge_id)
manifest.artifacts.update(artifacts)
if token_equivalent is not None:
manifest.token_equivalent = token_equivalent
manifest_path = root / "manifest.json"
manifest.dna = self.compute_dna(root)
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
self.db.write_cartridge(
cartridge_id=manifest.cartridge_id,
name=manifest.name,
root_path=root,
manifest_json=manifest.model_dump_json(),
dna=manifest.dna,
token_equivalent=manifest.token_equivalent,
created_at=manifest.created_at,
)
self._commit(manifest, root)
return manifest
def attach_eval_proof(self, cartridge_id: str, proof_dir: Path) -> CartridgeManifest:
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
if not rows:
raise ValueError(f"Unknown cartridge: {cartridge_id}")
root, manifest = self._require(cartridge_id)
if not proof_dir.exists() or not proof_dir.is_dir():
raise ValueError(f"Missing proof directory: {proof_dir}")
row = rows[0]
root = Path(row["root_path"])
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
target = root / "eval_proofs" / proof_dir.name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(proof_dir, target, dirs_exist_ok=True)
manifest.artifacts[f"eval_proof_{proof_dir.name}"] = target.relative_to(root).as_posix() + "/"
manifest_path = root / "manifest.json"
manifest.dna = self.compute_dna(root)
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
self.db.write_cartridge(
cartridge_id=manifest.cartridge_id,
name=manifest.name,
root_path=root,
manifest_json=manifest.model_dump_json(),
dna=manifest.dna,
token_equivalent=manifest.token_equivalent,
created_at=manifest.created_at,
)
self._commit(manifest, root)
return manifest
def _write_atoms_parquet(self, path: Path, atoms: list[EvidenceAtom]) -> None:
rows = [atom.model_dump(mode="json") for atom in atoms]
table = pa.Table.from_pylist(rows)
rows = []
for atom in atoms:
row = atom.model_dump(mode="json")
for field in JSON_PARQUET_COLUMNS:
row[field] = json.dumps(row[field], ensure_ascii=False, sort_keys=True)
rows.append(row)
# An empty struct column has no child field, and Parquet cannot write one: a memory
# atom carries metadata={} and would crash the export.
table = pa.Table.from_pylist(rows) if rows else pa.Table.from_pydict({"atom_id": []})
pq.write_table(table, path, compression="zstd")
def _write_sparse_index(self, path: Path, atoms: list[EvidenceAtom]) -> None:
@@ -289,13 +331,13 @@ class CartridgeService:
connection.close()
def _cartridge_id(self, name: str, created_at: datetime) -> str:
raw = f"{name}:{created_at.isoformat()}".encode("utf-8")
raw = f"{name}:{created_at.isoformat()}".encode()
safe_name = "".join(char if char.isalnum() else "_" for char in name.lower()).strip("_")[:40]
return f"{safe_name or 'cartridge'}_{hashlib.sha256(raw).hexdigest()[:12]}"
class _DigestWriter:
def __init__(self, digest: "hashlib._Hash") -> None:
def __init__(self, digest: hashlib._Hash) -> None:
self.digest = digest
def write(self, data: bytes) -> int: