Make a cartridge a scope you mount, not only a bundle you export
A cartridge used to be an export: materialize it, hash it, and never let it
change behaviour again. The transactive-memory promise needs the opposite —
mounting the home cartridge must make the work cartridge's memories
unfindable, and appending to a cartridge must make a new memory findable
through it.
Cartridge scope (schema v2 -> v3):
- cartridge_atoms (which atoms are in which cartridge) and cartridge_mounts
(which cartridges this machine has mounted now); both rebuilt on reopen,
because a mount is a machine-local state, not a fact about the data.
- mount / switch / unmount / append refuse a cartridge that was built for a
different runtime, naming every differing field instead of a boolean.
- append rewrites atoms.parquet and sparse_index.bm25.zst, recomputes DNA and
the KV fingerprint, and keeps the cartridges row in one SQLite transaction;
the files are written first so a failed export cannot leave a half-cartridge
in the database.
- RetrievalService.search() now ranks inside the mounted scope by default, and
an explicit cartridge_ids filter overrides the mount. Nothing mounted means
the whole library, so an empty store behaves as it did before.
- forget() removes the forgotten atoms from every cartridge's membership:
forgetting must not leave a memory retrievable through a cartridge.
KV fingerprint (kv_fingerprint.py, stored in the manifest):
- model / tokenizer / chat-template hashes, the RoPE block (factor,
original_max_position_embeddings, mrope_interleaved, mrope_section,
partial_rotary_factor), the adapter hash, the context length, the engine
version, and the fingerprint format version.
- A snapshot we cannot read is recorded as unverifiable and listed in
unverifiable_fields; a blind machine refuses to mount rather than pretending
the runtime matches, because a wrong RoPE factor silently corrupts retrieval.
Two defects found by running the new tests, not by reading:
- atoms.parquet could not export a memory at all: pyarrow refuses a struct with
no child field, so metadata={} failed the whole table. The six JSON columns
are now written as JSON text and decoded by the cartridge reader.
- materialize() inserted cartridge membership before the cartridges row existed,
so the foreign key rejected every cartridge.
api.py crossed 500 lines, so the cartridge and mount routes moved to
api_cartridges.py. That module must not use `from __future__ import
annotations`: FastAPI then resolves Depends(settings_dep) against module
globals, where the injected dependency is only a parameter of register(), and
every route starts demanding settings as a query parameter.
Verification: 138 pytest passed (26 new: 10 fingerprint, 16 scope), Go
build/vet/gofmt/test clean, and scripts/mcp_write_smoke.sh now proves the whole
loop across two processes - mount narrows search, a memory written after the
upload is invisible until append_to_cartridge, unmount restores the library,
and rewriting the RoPE factor makes mount_cartridge refuse with "rope: factor:
cartridge 4.0 vs local 2.0". ruff clean on every touched file (the 20 remaining
package errors are in files this change did not touch; HEAD had 25). mypy: one
error, the pre-existing np.savez stub artifact at cartridge_assets.py:329
(HEAD:320). Repo pyright: 62 errors, unchanged from the p0-3 ledger.
This commit is contained in:
1 parent
674c88e84e
commit
9fe13fffb3
21 files changed
+1492
-153
No files matched your search
@@ -4,16 +4,34 @@ import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
import zstandard as zstd
|
||||
|
||||
from . import cartridge_store
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .schemas import AdapterManifest, CartridgeManifest, EvidenceAtom, SourceRecord
|
||||
from .kv_fingerprint import compute_fingerprint, fingerprint_diff, unverifiable_fields
|
||||
from .schemas import (
|
||||
AdapterManifest,
|
||||
CartridgeManifest,
|
||||
EvidenceAtom,
|
||||
MountState,
|
||||
SourceRecord,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
JSON_PARQUET_COLUMNS = (
|
||||
"embedding_refs",
|
||||
"layout",
|
||||
"relations",
|
||||
"trust",
|
||||
"metadata",
|
||||
"visibility",
|
||||
)
|
||||
|
||||
|
||||
class CartridgeService:
|
||||
@@ -27,7 +45,7 @@ class CartridgeService:
|
||||
atoms: list[EvidenceAtom],
|
||||
sources: list[SourceRecord],
|
||||
) -> CartridgeManifest:
|
||||
created_at = datetime.now(timezone.utc)
|
||||
created_at = datetime.now(UTC)
|
||||
cartridge_id = self._cartridge_id(name, created_at)
|
||||
root = self.settings.cartridge_dir / cartridge_id
|
||||
eval_dir = root / "eval_proofs"
|
||||
@@ -55,6 +73,7 @@ class CartridgeService:
|
||||
source_count=len(sources),
|
||||
atom_count=len(atoms),
|
||||
token_equivalent=sum(atom.token_equivalent for atom in atoms),
|
||||
kv_fingerprint=compute_fingerprint(self.settings, root / "adapters"),
|
||||
artifacts={
|
||||
"manifest": "manifest.json",
|
||||
"atoms": "atoms.parquet",
|
||||
@@ -64,20 +83,8 @@ class CartridgeService:
|
||||
"eval_proofs": "eval_proofs/",
|
||||
},
|
||||
)
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
dna = self.compute_dna(root)
|
||||
manifest.dna = dna
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=cartridge_id,
|
||||
name=name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
cartridge_store.set_membership(self.db, cartridge_id, [atom.atom_id for atom in atoms])
|
||||
return manifest
|
||||
|
||||
def compute_dna(self, root: Path) -> str:
|
||||
@@ -96,18 +103,102 @@ class CartridgeService:
|
||||
manifests.append(CartridgeManifest.model_validate_json(row["manifest_json"]))
|
||||
return manifests
|
||||
|
||||
def mount_state(self) -> MountState:
|
||||
mounted = cartridge_store.mounted_ids(self.db)
|
||||
return MountState(
|
||||
mounted=mounted,
|
||||
scoped=bool(mounted),
|
||||
atoms_in_scope=len(cartridge_store.atoms_of_cartridges(self.db, mounted)),
|
||||
)
|
||||
|
||||
def mount(self, cartridge_id: str) -> CartridgeManifest:
|
||||
"""Add one cartridge to the mounted set; a mounted search answers only from it."""
|
||||
root, manifest = self._require(cartridge_id)
|
||||
self._refuse_if_incompatible(manifest, root)
|
||||
mounted = cartridge_store.mounted_ids(self.db)
|
||||
if cartridge_id not in mounted:
|
||||
cartridge_store.set_mounts(self.db, [*mounted, cartridge_id], utc_now())
|
||||
return manifest
|
||||
|
||||
def switch(self, cartridge_id: str) -> CartridgeManifest:
|
||||
"""Mount one cartridge and nothing else."""
|
||||
root, manifest = self._require(cartridge_id)
|
||||
self._refuse_if_incompatible(manifest, root)
|
||||
cartridge_store.set_mounts(self.db, [cartridge_id], utc_now())
|
||||
return manifest
|
||||
|
||||
def unmount(self, cartridge_id: str) -> MountState:
|
||||
self._require(cartridge_id)
|
||||
mounted = cartridge_store.mounted_ids(self.db)
|
||||
if cartridge_id not in mounted:
|
||||
raise ValueError(f"{cartridge_id} is not mounted; nothing to unmount")
|
||||
cartridge_store.set_mounts(
|
||||
self.db, [held for held in mounted if held != cartridge_id], utc_now()
|
||||
)
|
||||
return self.mount_state()
|
||||
|
||||
def append(self, cartridge_id: str, atom_ids: list[str]) -> CartridgeManifest:
|
||||
"""Add memories to an existing cartridge; the bundle and its DNA follow the membership."""
|
||||
self._require(cartridge_id)
|
||||
if not atom_ids:
|
||||
raise ValueError("append needs at least one atom")
|
||||
atoms = self.db.get_atoms(atom_ids)
|
||||
found = {atom.atom_id for atom in atoms}
|
||||
missing = [atom_id for atom_id in atom_ids if atom_id not in found]
|
||||
if missing:
|
||||
raise ValueError(f"cannot append, these atoms are not in the store: {', '.join(missing)}")
|
||||
cartridge_store.add_membership(self.db, cartridge_id, atom_ids)
|
||||
held = self.db.get_atoms(sorted(cartridge_store.atoms_of_cartridge(self.db, cartridge_id)))
|
||||
sources = self.db.get_sources(sorted({atom.source_id for atom in held}))
|
||||
return self.rebuild(cartridge_id, held, sources)
|
||||
|
||||
def _require(self, cartridge_id: str) -> tuple[Path, CartridgeManifest]:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
return Path(row["root_path"]), CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
|
||||
def _refuse_if_incompatible(self, manifest: CartridgeManifest, root: Path) -> None:
|
||||
"""KV bytes are only valid under the settings that produced them."""
|
||||
stored = manifest.kv_fingerprint
|
||||
if stored is None:
|
||||
raise ValueError(f"{manifest.cartridge_id} carries no kv fingerprint; rebuild it before mounting")
|
||||
live = compute_fingerprint(self.settings, root / "adapters")
|
||||
gaps = unverifiable_fields(live)
|
||||
if gaps:
|
||||
raise ValueError(
|
||||
f"cannot verify {manifest.cartridge_id} on this machine: {', '.join(gaps)} are unknown; "
|
||||
"set ELY_EYE_MODEL_DIR to the model snapshot"
|
||||
)
|
||||
differences = fingerprint_diff(stored, live)
|
||||
if differences:
|
||||
raise ValueError(
|
||||
f"{manifest.cartridge_id} was built for a different runtime; refusing to mount: "
|
||||
+ "; ".join(differences)
|
||||
)
|
||||
|
||||
def _commit(self, manifest: CartridgeManifest, root: Path) -> None:
|
||||
"""Re-hash the bundle, then record the manifest that describes those bytes."""
|
||||
manifest.dna = self.compute_dna(root)
|
||||
(root / "manifest.json").write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
|
||||
def rebuild(
|
||||
self,
|
||||
cartridge_id: str,
|
||||
atoms: list[EvidenceAtom],
|
||||
sources: list[SourceRecord],
|
||||
) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
root, manifest = self._require(cartridge_id)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._write_atoms_parquet(root / "atoms.parquet", atoms)
|
||||
@@ -133,24 +224,11 @@ class CartridgeService:
|
||||
)
|
||||
manifest.dna = self.compute_dna(root)
|
||||
(root / "manifest.json").write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def attach_adapter(self, cartridge_id: str, adapter_dir: Path) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
root, manifest = self._require(cartridge_id)
|
||||
adapter_manifest_path = adapter_dir / "adapter_manifest.json"
|
||||
if not adapter_manifest_path.exists():
|
||||
raise ValueError(f"Missing adapter manifest: {adapter_manifest_path}")
|
||||
@@ -163,18 +241,7 @@ class CartridgeService:
|
||||
manifest.artifacts[f"adapter_{adapter_manifest.kind.value}"] = (
|
||||
adapter_root.relative_to(root).as_posix() + "/"
|
||||
)
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest.dna = self.compute_dna(root)
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def update_artifacts(
|
||||
@@ -183,59 +250,34 @@ class CartridgeService:
|
||||
artifacts: dict[str, str],
|
||||
token_equivalent: int | None = None,
|
||||
) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
root, manifest = self._require(cartridge_id)
|
||||
manifest.artifacts.update(artifacts)
|
||||
if token_equivalent is not None:
|
||||
manifest.token_equivalent = token_equivalent
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest.dna = self.compute_dna(root)
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def attach_eval_proof(self, cartridge_id: str, proof_dir: Path) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
root, manifest = self._require(cartridge_id)
|
||||
if not proof_dir.exists() or not proof_dir.is_dir():
|
||||
raise ValueError(f"Missing proof directory: {proof_dir}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
target = root / "eval_proofs" / proof_dir.name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(proof_dir, target, dirs_exist_ok=True)
|
||||
manifest.artifacts[f"eval_proof_{proof_dir.name}"] = target.relative_to(root).as_posix() + "/"
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest.dna = self.compute_dna(root)
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def _write_atoms_parquet(self, path: Path, atoms: list[EvidenceAtom]) -> None:
|
||||
rows = [atom.model_dump(mode="json") for atom in atoms]
|
||||
table = pa.Table.from_pylist(rows)
|
||||
rows = []
|
||||
for atom in atoms:
|
||||
row = atom.model_dump(mode="json")
|
||||
for field in JSON_PARQUET_COLUMNS:
|
||||
row[field] = json.dumps(row[field], ensure_ascii=False, sort_keys=True)
|
||||
rows.append(row)
|
||||
# An empty struct column has no child field, and Parquet cannot write one: a memory
|
||||
# atom carries metadata={} and would crash the export.
|
||||
table = pa.Table.from_pylist(rows) if rows else pa.Table.from_pydict({"atom_id": []})
|
||||
pq.write_table(table, path, compression="zstd")
|
||||
|
||||
def _write_sparse_index(self, path: Path, atoms: list[EvidenceAtom]) -> None:
|
||||
@@ -289,13 +331,13 @@ class CartridgeService:
|
||||
connection.close()
|
||||
|
||||
def _cartridge_id(self, name: str, created_at: datetime) -> str:
|
||||
raw = f"{name}:{created_at.isoformat()}".encode("utf-8")
|
||||
raw = f"{name}:{created_at.isoformat()}".encode()
|
||||
safe_name = "".join(char if char.isalnum() else "_" for char in name.lower()).strip("_")[:40]
|
||||
return f"{safe_name or 'cartridge'}_{hashlib.sha256(raw).hexdigest()[:12]}"
|
||||
|
||||
|
||||
class _DigestWriter:
|
||||
def __init__(self, digest: "hashlib._Hash") -> None:
|
||||
def __init__(self, digest: hashlib._Hash) -> None:
|
||||
self.digest = digest
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
|
||||
Reference in new issue
Block a user