Make a cartridge a scope you mount, not only a bundle you export
A cartridge used to be an export: materialize it, hash it, and never let it
change behaviour again. The transactive-memory promise needs the opposite —
mounting the home cartridge must make the work cartridge's memories
unfindable, and appending to a cartridge must make a new memory findable
through it.
Cartridge scope (schema v2 -> v3):
- cartridge_atoms (which atoms are in which cartridge) and cartridge_mounts
(which cartridges this machine has mounted now); both rebuilt on reopen,
because a mount is a machine-local state, not a fact about the data.
- mount / switch / unmount / append refuse a cartridge that was built for a
different runtime, naming every differing field instead of a boolean.
- append rewrites atoms.parquet and sparse_index.bm25.zst, recomputes DNA and
the KV fingerprint, and keeps the cartridges row in one SQLite transaction;
the files are written first so a failed export cannot leave a half-cartridge
in the database.
- RetrievalService.search() now ranks inside the mounted scope by default, and
an explicit cartridge_ids filter overrides the mount. Nothing mounted means
the whole library, so an empty store behaves as it did before.
- forget() removes the forgotten atoms from every cartridge's membership:
forgetting must not leave a memory retrievable through a cartridge.
KV fingerprint (kv_fingerprint.py, stored in the manifest):
- model / tokenizer / chat-template hashes, the RoPE block (factor,
original_max_position_embeddings, mrope_interleaved, mrope_section,
partial_rotary_factor), the adapter hash, the context length, the engine
version, and the fingerprint format version.
- A snapshot we cannot read is recorded as unverifiable and listed in
unverifiable_fields; a blind machine refuses to mount rather than pretending
the runtime matches, because a wrong RoPE factor silently corrupts retrieval.
Two defects found by running the new tests, not by reading:
- atoms.parquet could not export a memory at all: pyarrow refuses a struct with
no child field, so metadata={} failed the whole table. The six JSON columns
are now written as JSON text and decoded by the cartridge reader.
- materialize() inserted cartridge membership before the cartridges row existed,
so the foreign key rejected every cartridge.
api.py crossed 500 lines, so the cartridge and mount routes moved to
api_cartridges.py. That module must not use `from __future__ import
annotations`: FastAPI then resolves Depends(settings_dep) against module
globals, where the injected dependency is only a parameter of register(), and
every route starts demanding settings as a query parameter.
Verification: 138 pytest passed (26 new: 10 fingerprint, 16 scope), Go
build/vet/gofmt/test clean, and scripts/mcp_write_smoke.sh now proves the whole
loop across two processes - mount narrows search, a memory written after the
upload is invisible until append_to_cartridge, unmount restores the library,
and rewriting the RoPE factor makes mount_cartridge refuse with "rope: factor:
cartridge 4.0 vs local 2.0". ruff clean on every touched file (the 20 remaining
package errors are in files this change did not touch; HEAD had 25). mypy: one
error, the pre-existing np.savez stub artifact at cartridge_assets.py:329
(HEAD:320). Repo pyright: 62 errors, unchanged from the p0-3 ledger.
This commit is contained in:
1 parent
674c88e84e
commit
9fe13fffb3
21 files changed
+1492
-153
No files matched your search
@@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel
|
||||
|
||||
from . import __version__
|
||||
from .api_cartridges import register as register_cartridge_routes
|
||||
from .benchmarks import (
|
||||
benchmark_prediction_report,
|
||||
benchmark_registry_report,
|
||||
@@ -18,7 +19,6 @@ from .benchmarks import (
|
||||
)
|
||||
from .cache_fabric import CacheFabric
|
||||
from .cartridge import CartridgeService
|
||||
from .cartridge_assets import CartridgeAssetService
|
||||
from .compiler import ContextCompiler
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
@@ -44,7 +44,6 @@ from .schemas import (
|
||||
AdapterManifest,
|
||||
AdapterStatus,
|
||||
AnswerResponse,
|
||||
CartridgeAssetReport,
|
||||
EvidenceAtom,
|
||||
ForgetReport,
|
||||
ForgetRequest,
|
||||
@@ -246,47 +245,7 @@ def create_app() -> FastAPI:
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
@app.get("/api/cartridges")
|
||||
def cartridges(settings: Annotated[Settings, Depends(settings_dep)]):
|
||||
return CartridgeService(settings).list_manifests()
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/finalize", response_model=CartridgeAssetReport)
|
||||
def finalize_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeAssetReport:
|
||||
try:
|
||||
return CartridgeAssetService(settings).finalize(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/refresh")
|
||||
def refresh_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
):
|
||||
db = Database(settings)
|
||||
try:
|
||||
return CartridgeService(settings, db).rebuild(
|
||||
cartridge_id=cartridge_id,
|
||||
atoms=db.list_atoms(),
|
||||
sources=db.list_sources(),
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get("/api/cartridges/{cartridge_id}/assets", response_model=CartridgeAssetReport)
|
||||
def cartridge_assets(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeAssetReport:
|
||||
try:
|
||||
report = CartridgeAssetService(settings).status(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if report is None:
|
||||
raise HTTPException(status_code=404, detail="Cartridge assets have not been finalized.")
|
||||
return report
|
||||
register_cartridge_routes(app, settings_dep)
|
||||
|
||||
@app.get("/api/adapters", response_model=list[AdapterStatus])
|
||||
def adapters(settings: Annotated[Settings, Depends(settings_dep)]) -> list[AdapterStatus]:
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Cartridge and mount routes: a cartridge is a retrieval scope, not only an export."""
|
||||
|
||||
# No `from __future__ import annotations` here: FastAPI reads the route signatures, and
|
||||
# string annotations would be resolved against this module's globals, where the injected
|
||||
# settings_dep lives only as a parameter of register().
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException
|
||||
|
||||
from .cartridge import CartridgeService
|
||||
from .cartridge_assets import CartridgeAssetService
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .schemas import AppendRequest, CartridgeAssetReport, CartridgeManifest, MountState
|
||||
|
||||
|
||||
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
"""Mounting decides what a search may see; the KV fingerprint decides if it may mount here."""
|
||||
|
||||
@app.get("/api/cartridges")
|
||||
def cartridges(settings: Annotated[Settings, Depends(settings_dep)]):
|
||||
return CartridgeService(settings).list_manifests()
|
||||
|
||||
@app.get("/api/mounts", response_model=MountState)
|
||||
def mounts(settings: Annotated[Settings, Depends(settings_dep)]) -> MountState:
|
||||
return CartridgeService(settings).mount_state()
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/mount", response_model=CartridgeManifest)
|
||||
def mount_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeManifest:
|
||||
try:
|
||||
return CartridgeService(settings).mount(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/switch", response_model=CartridgeManifest)
|
||||
def switch_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeManifest:
|
||||
try:
|
||||
return CartridgeService(settings).switch(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/unmount", response_model=MountState)
|
||||
def unmount_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> MountState:
|
||||
try:
|
||||
return CartridgeService(settings).unmount(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/append", response_model=CartridgeManifest)
|
||||
def append_cartridge(
|
||||
cartridge_id: str,
|
||||
request: AppendRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeManifest:
|
||||
try:
|
||||
return CartridgeService(settings).append(cartridge_id, request.atom_ids)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/finalize", response_model=CartridgeAssetReport)
|
||||
def finalize_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeAssetReport:
|
||||
try:
|
||||
return CartridgeAssetService(settings).finalize(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/cartridges/{cartridge_id}/refresh")
|
||||
def refresh_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
):
|
||||
db = Database(settings)
|
||||
try:
|
||||
return CartridgeService(settings, db).rebuild(
|
||||
cartridge_id=cartridge_id,
|
||||
atoms=db.list_atoms(),
|
||||
sources=db.list_sources(),
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
@app.get("/api/cartridges/{cartridge_id}/assets", response_model=CartridgeAssetReport)
|
||||
def cartridge_assets(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> CartridgeAssetReport:
|
||||
try:
|
||||
report = CartridgeAssetService(settings).status(cartridge_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if report is None:
|
||||
raise HTTPException(status_code=404, detail="Cartridge assets have not been finalized.")
|
||||
return report
|
||||
@@ -4,16 +4,34 @@ import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
import zstandard as zstd
|
||||
|
||||
from . import cartridge_store
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .schemas import AdapterManifest, CartridgeManifest, EvidenceAtom, SourceRecord
|
||||
from .kv_fingerprint import compute_fingerprint, fingerprint_diff, unverifiable_fields
|
||||
from .schemas import (
|
||||
AdapterManifest,
|
||||
CartridgeManifest,
|
||||
EvidenceAtom,
|
||||
MountState,
|
||||
SourceRecord,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
JSON_PARQUET_COLUMNS = (
|
||||
"embedding_refs",
|
||||
"layout",
|
||||
"relations",
|
||||
"trust",
|
||||
"metadata",
|
||||
"visibility",
|
||||
)
|
||||
|
||||
|
||||
class CartridgeService:
|
||||
@@ -27,7 +45,7 @@ class CartridgeService:
|
||||
atoms: list[EvidenceAtom],
|
||||
sources: list[SourceRecord],
|
||||
) -> CartridgeManifest:
|
||||
created_at = datetime.now(timezone.utc)
|
||||
created_at = datetime.now(UTC)
|
||||
cartridge_id = self._cartridge_id(name, created_at)
|
||||
root = self.settings.cartridge_dir / cartridge_id
|
||||
eval_dir = root / "eval_proofs"
|
||||
@@ -55,6 +73,7 @@ class CartridgeService:
|
||||
source_count=len(sources),
|
||||
atom_count=len(atoms),
|
||||
token_equivalent=sum(atom.token_equivalent for atom in atoms),
|
||||
kv_fingerprint=compute_fingerprint(self.settings, root / "adapters"),
|
||||
artifacts={
|
||||
"manifest": "manifest.json",
|
||||
"atoms": "atoms.parquet",
|
||||
@@ -64,20 +83,8 @@ class CartridgeService:
|
||||
"eval_proofs": "eval_proofs/",
|
||||
},
|
||||
)
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
dna = self.compute_dna(root)
|
||||
manifest.dna = dna
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=cartridge_id,
|
||||
name=name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
cartridge_store.set_membership(self.db, cartridge_id, [atom.atom_id for atom in atoms])
|
||||
return manifest
|
||||
|
||||
def compute_dna(self, root: Path) -> str:
|
||||
@@ -96,18 +103,102 @@ class CartridgeService:
|
||||
manifests.append(CartridgeManifest.model_validate_json(row["manifest_json"]))
|
||||
return manifests
|
||||
|
||||
def mount_state(self) -> MountState:
|
||||
mounted = cartridge_store.mounted_ids(self.db)
|
||||
return MountState(
|
||||
mounted=mounted,
|
||||
scoped=bool(mounted),
|
||||
atoms_in_scope=len(cartridge_store.atoms_of_cartridges(self.db, mounted)),
|
||||
)
|
||||
|
||||
def mount(self, cartridge_id: str) -> CartridgeManifest:
|
||||
"""Add one cartridge to the mounted set; a mounted search answers only from it."""
|
||||
root, manifest = self._require(cartridge_id)
|
||||
self._refuse_if_incompatible(manifest, root)
|
||||
mounted = cartridge_store.mounted_ids(self.db)
|
||||
if cartridge_id not in mounted:
|
||||
cartridge_store.set_mounts(self.db, [*mounted, cartridge_id], utc_now())
|
||||
return manifest
|
||||
|
||||
def switch(self, cartridge_id: str) -> CartridgeManifest:
|
||||
"""Mount one cartridge and nothing else."""
|
||||
root, manifest = self._require(cartridge_id)
|
||||
self._refuse_if_incompatible(manifest, root)
|
||||
cartridge_store.set_mounts(self.db, [cartridge_id], utc_now())
|
||||
return manifest
|
||||
|
||||
def unmount(self, cartridge_id: str) -> MountState:
|
||||
self._require(cartridge_id)
|
||||
mounted = cartridge_store.mounted_ids(self.db)
|
||||
if cartridge_id not in mounted:
|
||||
raise ValueError(f"{cartridge_id} is not mounted; nothing to unmount")
|
||||
cartridge_store.set_mounts(
|
||||
self.db, [held for held in mounted if held != cartridge_id], utc_now()
|
||||
)
|
||||
return self.mount_state()
|
||||
|
||||
def append(self, cartridge_id: str, atom_ids: list[str]) -> CartridgeManifest:
|
||||
"""Add memories to an existing cartridge; the bundle and its DNA follow the membership."""
|
||||
self._require(cartridge_id)
|
||||
if not atom_ids:
|
||||
raise ValueError("append needs at least one atom")
|
||||
atoms = self.db.get_atoms(atom_ids)
|
||||
found = {atom.atom_id for atom in atoms}
|
||||
missing = [atom_id for atom_id in atom_ids if atom_id not in found]
|
||||
if missing:
|
||||
raise ValueError(f"cannot append, these atoms are not in the store: {', '.join(missing)}")
|
||||
cartridge_store.add_membership(self.db, cartridge_id, atom_ids)
|
||||
held = self.db.get_atoms(sorted(cartridge_store.atoms_of_cartridge(self.db, cartridge_id)))
|
||||
sources = self.db.get_sources(sorted({atom.source_id for atom in held}))
|
||||
return self.rebuild(cartridge_id, held, sources)
|
||||
|
||||
def _require(self, cartridge_id: str) -> tuple[Path, CartridgeManifest]:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
return Path(row["root_path"]), CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
|
||||
def _refuse_if_incompatible(self, manifest: CartridgeManifest, root: Path) -> None:
|
||||
"""KV bytes are only valid under the settings that produced them."""
|
||||
stored = manifest.kv_fingerprint
|
||||
if stored is None:
|
||||
raise ValueError(f"{manifest.cartridge_id} carries no kv fingerprint; rebuild it before mounting")
|
||||
live = compute_fingerprint(self.settings, root / "adapters")
|
||||
gaps = unverifiable_fields(live)
|
||||
if gaps:
|
||||
raise ValueError(
|
||||
f"cannot verify {manifest.cartridge_id} on this machine: {', '.join(gaps)} are unknown; "
|
||||
"set ELY_EYE_MODEL_DIR to the model snapshot"
|
||||
)
|
||||
differences = fingerprint_diff(stored, live)
|
||||
if differences:
|
||||
raise ValueError(
|
||||
f"{manifest.cartridge_id} was built for a different runtime; refusing to mount: "
|
||||
+ "; ".join(differences)
|
||||
)
|
||||
|
||||
def _commit(self, manifest: CartridgeManifest, root: Path) -> None:
|
||||
"""Re-hash the bundle, then record the manifest that describes those bytes."""
|
||||
manifest.dna = self.compute_dna(root)
|
||||
(root / "manifest.json").write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
|
||||
def rebuild(
|
||||
self,
|
||||
cartridge_id: str,
|
||||
atoms: list[EvidenceAtom],
|
||||
sources: list[SourceRecord],
|
||||
) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
root, manifest = self._require(cartridge_id)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._write_atoms_parquet(root / "atoms.parquet", atoms)
|
||||
@@ -133,24 +224,11 @@ class CartridgeService:
|
||||
)
|
||||
manifest.dna = self.compute_dna(root)
|
||||
(root / "manifest.json").write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def attach_adapter(self, cartridge_id: str, adapter_dir: Path) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
root, manifest = self._require(cartridge_id)
|
||||
adapter_manifest_path = adapter_dir / "adapter_manifest.json"
|
||||
if not adapter_manifest_path.exists():
|
||||
raise ValueError(f"Missing adapter manifest: {adapter_manifest_path}")
|
||||
@@ -163,18 +241,7 @@ class CartridgeService:
|
||||
manifest.artifacts[f"adapter_{adapter_manifest.kind.value}"] = (
|
||||
adapter_root.relative_to(root).as_posix() + "/"
|
||||
)
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest.dna = self.compute_dna(root)
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def update_artifacts(
|
||||
@@ -183,59 +250,34 @@ class CartridgeService:
|
||||
artifacts: dict[str, str],
|
||||
token_equivalent: int | None = None,
|
||||
) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
root, manifest = self._require(cartridge_id)
|
||||
manifest.artifacts.update(artifacts)
|
||||
if token_equivalent is not None:
|
||||
manifest.token_equivalent = token_equivalent
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest.dna = self.compute_dna(root)
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def attach_eval_proof(self, cartridge_id: str, proof_dir: Path) -> CartridgeManifest:
|
||||
rows = [row for row in self.db.list_cartridges() if row["cartridge_id"] == cartridge_id]
|
||||
if not rows:
|
||||
raise ValueError(f"Unknown cartridge: {cartridge_id}")
|
||||
root, manifest = self._require(cartridge_id)
|
||||
if not proof_dir.exists() or not proof_dir.is_dir():
|
||||
raise ValueError(f"Missing proof directory: {proof_dir}")
|
||||
row = rows[0]
|
||||
root = Path(row["root_path"])
|
||||
manifest = CartridgeManifest.model_validate_json(row["manifest_json"])
|
||||
target = root / "eval_proofs" / proof_dir.name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(proof_dir, target, dirs_exist_ok=True)
|
||||
manifest.artifacts[f"eval_proof_{proof_dir.name}"] = target.relative_to(root).as_posix() + "/"
|
||||
manifest_path = root / "manifest.json"
|
||||
manifest.dna = self.compute_dna(root)
|
||||
manifest_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
|
||||
self.db.write_cartridge(
|
||||
cartridge_id=manifest.cartridge_id,
|
||||
name=manifest.name,
|
||||
root_path=root,
|
||||
manifest_json=manifest.model_dump_json(),
|
||||
dna=manifest.dna,
|
||||
token_equivalent=manifest.token_equivalent,
|
||||
created_at=manifest.created_at,
|
||||
)
|
||||
self._commit(manifest, root)
|
||||
return manifest
|
||||
|
||||
def _write_atoms_parquet(self, path: Path, atoms: list[EvidenceAtom]) -> None:
|
||||
rows = [atom.model_dump(mode="json") for atom in atoms]
|
||||
table = pa.Table.from_pylist(rows)
|
||||
rows = []
|
||||
for atom in atoms:
|
||||
row = atom.model_dump(mode="json")
|
||||
for field in JSON_PARQUET_COLUMNS:
|
||||
row[field] = json.dumps(row[field], ensure_ascii=False, sort_keys=True)
|
||||
rows.append(row)
|
||||
# An empty struct column has no child field, and Parquet cannot write one: a memory
|
||||
# atom carries metadata={} and would crash the export.
|
||||
table = pa.Table.from_pylist(rows) if rows else pa.Table.from_pydict({"atom_id": []})
|
||||
pq.write_table(table, path, compression="zstd")
|
||||
|
||||
def _write_sparse_index(self, path: Path, atoms: list[EvidenceAtom]) -> None:
|
||||
@@ -289,13 +331,13 @@ class CartridgeService:
|
||||
connection.close()
|
||||
|
||||
def _cartridge_id(self, name: str, created_at: datetime) -> str:
|
||||
raw = f"{name}:{created_at.isoformat()}".encode("utf-8")
|
||||
raw = f"{name}:{created_at.isoformat()}".encode()
|
||||
safe_name = "".join(char if char.isalnum() else "_" for char in name.lower()).strip("_")[:40]
|
||||
return f"{safe_name or 'cartridge'}_{hashlib.sha256(raw).hexdigest()[:12]}"
|
||||
|
||||
|
||||
class _DigestWriter:
|
||||
def __init__(self, digest: "hashlib._Hash") -> None:
|
||||
def __init__(self, digest: hashlib._Hash) -> None:
|
||||
self.digest = digest
|
||||
|
||||
def write(self, data: bytes) -> int:
|
||||
|
||||
@@ -14,13 +14,12 @@ import pyarrow.parquet as pq
|
||||
import zstandard as zstd
|
||||
from PIL import Image
|
||||
|
||||
from .cartridge import CartridgeService
|
||||
from .cartridge import JSON_PARQUET_COLUMNS, CartridgeService
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .schemas import CartridgeAssetReport, CartridgeAssetStatus
|
||||
from .storage import ObjectStore, sha256_file
|
||||
|
||||
|
||||
TEXT_VECTOR_DIMS = 384
|
||||
VISUAL_VECTOR_DIMS = 256
|
||||
MEMORY_CAPSULE_SEGMENTS = 4096
|
||||
@@ -269,7 +268,17 @@ class CartridgeAssetService:
|
||||
def load_cartridge_atoms(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(path)
|
||||
return pq.read_table(path).to_pylist()
|
||||
rows = pq.read_table(path).to_pylist()
|
||||
for row in rows:
|
||||
for field in JSON_PARQUET_COLUMNS:
|
||||
try:
|
||||
row[field] = json.loads(row[field])
|
||||
except (TypeError, json.JSONDecodeError) as error:
|
||||
raise ValueError(
|
||||
f"{path}: column {field!r} is not readable JSON text ({error}); "
|
||||
"rebuild the cartridge"
|
||||
) from error
|
||||
return rows
|
||||
|
||||
|
||||
def hashed_text_vector(text: str, dims: int) -> np.ndarray:
|
||||
@@ -298,7 +307,7 @@ def visual_image_vector(path: Path, dims: int) -> np.ndarray:
|
||||
|
||||
|
||||
def capsule_hash(seed: str, index: int, label: str) -> str:
|
||||
return hashlib.sha256(f"{seed}:{index}:{label}".encode("utf-8")).hexdigest()
|
||||
return hashlib.sha256(f"{seed}:{index}:{label}".encode()).hexdigest()
|
||||
|
||||
|
||||
def merkle_root(hashes: list[str]) -> str:
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Membership and mount state for cartridges, kept out of db.py by the size rule.
|
||||
|
||||
A cartridge used to exist only as files copied out of the store, so "mount this cartridge"
|
||||
had nothing to filter against. Membership now lives beside the atoms, and the mount table
|
||||
is what a search scopes itself to when the caller does not name a scope.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
|
||||
from .db import Database
|
||||
from .row_codec import to_json
|
||||
from .schemas import utc_now
|
||||
|
||||
|
||||
def set_membership(database: Database, cartridge_id: str, atom_ids: list[str]) -> None:
|
||||
"""Replace the atom set a cartridge holds."""
|
||||
added_at = utc_now().isoformat()
|
||||
with database.connect() as connection:
|
||||
connection.execute("DELETE FROM cartridge_atoms WHERE cartridge_id = ?", (cartridge_id,))
|
||||
connection.executemany(
|
||||
"INSERT OR IGNORE INTO cartridge_atoms(cartridge_id, atom_id, added_at) VALUES (?, ?, ?)",
|
||||
[(cartridge_id, atom_id, added_at) for atom_id in atom_ids],
|
||||
)
|
||||
|
||||
|
||||
def add_membership(database: Database, cartridge_id: str, atom_ids: list[str]) -> int:
|
||||
"""Add atoms to a cartridge; returns how many were new."""
|
||||
added_at = utc_now().isoformat()
|
||||
with database.connect() as connection:
|
||||
before = _count(connection, cartridge_id)
|
||||
connection.executemany(
|
||||
"INSERT OR IGNORE INTO cartridge_atoms(cartridge_id, atom_id, added_at) VALUES (?, ?, ?)",
|
||||
[(cartridge_id, atom_id, added_at) for atom_id in atom_ids],
|
||||
)
|
||||
after = _count(connection, cartridge_id)
|
||||
return after - before
|
||||
|
||||
|
||||
def membership_count(database: Database, cartridge_id: str) -> int:
|
||||
with database.connect() as connection:
|
||||
return _count(connection, cartridge_id)
|
||||
|
||||
|
||||
def cartridges_holding_atom(database: Database, atom_id: str) -> list[str]:
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT cartridge_id FROM cartridge_atoms WHERE atom_id = ? ORDER BY cartridge_id",
|
||||
(atom_id,),
|
||||
).fetchall()
|
||||
return [str(row["cartridge_id"]) for row in rows]
|
||||
|
||||
|
||||
def atoms_of_cartridge(database: Database, cartridge_id: str) -> list[str]:
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT atom_id FROM cartridge_atoms WHERE cartridge_id = ? ORDER BY atom_id",
|
||||
(cartridge_id,),
|
||||
).fetchall()
|
||||
return [str(row["atom_id"]) for row in rows]
|
||||
|
||||
|
||||
def atoms_of_cartridges(database: Database, cartridge_ids: list[str]) -> set[str]:
|
||||
"""Every atom the named cartridges hold; the scope a search must not leak outside."""
|
||||
if not cartridge_ids:
|
||||
return set()
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT DISTINCT atom_id FROM cartridge_atoms "
|
||||
"WHERE cartridge_id IN (SELECT value FROM json_each(?))",
|
||||
(to_json(cartridge_ids),),
|
||||
).fetchall()
|
||||
return {str(row["atom_id"]) for row in rows}
|
||||
|
||||
|
||||
def mounted_ids(database: Database) -> list[str]:
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT cartridge_id, mounted_at FROM mounts ORDER BY mounted_at, cartridge_id"
|
||||
).fetchall()
|
||||
return [str(row["cartridge_id"]) for row in rows]
|
||||
|
||||
|
||||
def set_mounts(database: Database, cartridge_ids: list[str], mounted_at: datetime) -> None:
|
||||
"""Replace the mounted set; mounting is a whole-state write, not a toggle."""
|
||||
stamp = mounted_at.isoformat()
|
||||
with database.connect() as connection:
|
||||
connection.execute("DELETE FROM mounts")
|
||||
connection.executemany(
|
||||
"INSERT INTO mounts(cartridge_id, mounted_at) VALUES (?, ?)",
|
||||
[(cartridge_id, stamp) for cartridge_id in cartridge_ids],
|
||||
)
|
||||
|
||||
|
||||
def clear_mounts(database: Database) -> None:
|
||||
with database.connect() as connection:
|
||||
connection.execute("DELETE FROM mounts")
|
||||
|
||||
|
||||
def _count(connection: sqlite3.Connection, cartridge_id: str) -> int:
|
||||
row = connection.execute(
|
||||
"SELECT COUNT(*) AS n FROM cartridge_atoms WHERE cartridge_id = ?", (cartridge_id,)
|
||||
).fetchone()
|
||||
return int(row["n"])
|
||||
@@ -168,7 +168,7 @@ def forget(atom_id: str = "", source_id: str = "", reason: str = "") -> None:
|
||||
def build_dense_index() -> None:
|
||||
index = EmbeddingService(get_settings()).build_index()
|
||||
dimensions = int(index.vectors.shape[1]) if index.vectors.size else 0
|
||||
typer.echo({"atoms": len(index.atom_ids), "dimensions": dimensions})
|
||||
typer.echo(json_dump({"atoms": len(index.atom_ids), "dimensions": dimensions}))
|
||||
|
||||
|
||||
@app.command("list-cartridges")
|
||||
@@ -177,6 +177,38 @@ def list_cartridges() -> None:
|
||||
typer.echo(manifest.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command("mounts")
|
||||
def mounts() -> None:
|
||||
typer.echo(CartridgeService(get_settings()).mount_state().model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def mount(cartridge_id: str) -> None:
|
||||
"""Add a cartridge to the mounted set; a mounted search answers only from it."""
|
||||
typer.echo(CartridgeService(get_settings()).mount(cartridge_id).model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def switch(cartridge_id: str) -> None:
|
||||
"""Mount one cartridge and nothing else."""
|
||||
typer.echo(CartridgeService(get_settings()).switch(cartridge_id).model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def unmount(cartridge_id: str) -> None:
|
||||
typer.echo(CartridgeService(get_settings()).unmount(cartridge_id).model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def append(
|
||||
cartridge_id: str,
|
||||
atom_id: list[str] = typer.Option(..., "--atom-id"), # noqa: B008 # typer reads option defaults itself
|
||||
) -> None:
|
||||
"""Add memories to an existing cartridge; the bundle and its DNA follow."""
|
||||
service = CartridgeService(get_settings())
|
||||
typer.echo(service.append(cartridge_id, list(atom_id)).model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command("hashhop")
|
||||
def hashhop(kind: str = "visual", hops: int = 2, token_equivalent: int = 262_144) -> None:
|
||||
evaluator = HashHopEvaluator(get_settings())
|
||||
|
||||
@@ -14,6 +14,8 @@ class Settings(BaseSettings):
|
||||
home: Path = Field(default_factory=lambda: Path.cwd() / ".ely_eye")
|
||||
default_holder: str = "local"
|
||||
model_id: str = "Qwen/Qwen3.5-9B"
|
||||
model_dir: Path | None = None
|
||||
kv_cache_dtype: str = "auto"
|
||||
planner_model_id: str = "Qwen/Qwen3.5-0.8B"
|
||||
embedding_model_id: str = "Qwen/Qwen3-VL-Embedding-8B"
|
||||
adapter_dir: Path | None = None
|
||||
|
||||
@@ -187,6 +187,7 @@ class Database:
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
min_trust: float | None = None,
|
||||
cartridge_ids: list[str] | None = None,
|
||||
) -> FtsPage:
|
||||
"""BM25 over the FTS index joined to memories that are open right now.
|
||||
|
||||
@@ -215,6 +216,11 @@ class Database:
|
||||
json_extract(a.trust_json, '$.model')
|
||||
) >= ?
|
||||
))
|
||||
AND (? IS NULL OR EXISTS (
|
||||
SELECT 1 FROM cartridge_atoms AS ca
|
||||
WHERE ca.atom_id = f.atom_id
|
||||
AND ca.cartridge_id IN (SELECT value FROM json_each(?))
|
||||
))
|
||||
ORDER BY score
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
@@ -234,6 +240,8 @@ class Database:
|
||||
None if until is None else until.isoformat(),
|
||||
min_trust,
|
||||
min_trust,
|
||||
None if cartridge_ids is None else to_json(cartridge_ids),
|
||||
None if cartridge_ids is None else to_json(cartridge_ids),
|
||||
limit + 1,
|
||||
offset,
|
||||
),
|
||||
@@ -330,6 +338,17 @@ class Database:
|
||||
).fetchall()
|
||||
return [row_to_source(row) for row in rows]
|
||||
|
||||
def get_sources(self, source_ids: list[str]) -> list[SourceRecord]:
|
||||
if not source_ids:
|
||||
return []
|
||||
with self.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT source_id, path, kind, mime, sha256, size_bytes, created_at, metadata_json "
|
||||
"FROM sources WHERE source_id IN (SELECT value FROM json_each(?))",
|
||||
(to_json(source_ids),),
|
||||
).fetchall()
|
||||
return [row_to_source(row) for row in rows]
|
||||
|
||||
def cartridge_count(self) -> int:
|
||||
with self.connect() as connection:
|
||||
return int(connection.execute("SELECT COUNT(*) FROM cartridges").fetchone()[0])
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""The compatibility header a cartridge carries, and the check that gates a mount.
|
||||
|
||||
KV bytes are only valid under the RoPE, quantization and engine settings that produced
|
||||
them (profiles.py launches YaRN factor 4.0 and llama.cpp --cache-type-k q4_0). A cartridge
|
||||
mounted under different settings does not fail: it silently attends to wrong positions. So
|
||||
the manifest carries a fingerprint and the mount compares it field by field.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from importlib import metadata
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .config import Settings
|
||||
|
||||
FORMAT_SEMVER = "1.0.0"
|
||||
UNKNOWN = ""
|
||||
|
||||
|
||||
class KvFingerprint(BaseModel):
|
||||
format_semver: str = FORMAT_SEMVER
|
||||
model_id: str = UNKNOWN
|
||||
model_hash: str = UNKNOWN
|
||||
tokenizer_hash: str = UNKNOWN
|
||||
chat_template_hash: str = UNKNOWN
|
||||
rope: dict[str, Any] = {}
|
||||
context_tokens: int = 0
|
||||
kv_dtype: str = UNKNOWN
|
||||
engine: str = UNKNOWN
|
||||
engine_version: str = UNKNOWN
|
||||
adapter_hashes: dict[str, str] = {}
|
||||
|
||||
|
||||
def _read_bytes(path: Path) -> bytes | None:
|
||||
if not path.is_file():
|
||||
return None
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def _hex(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _config_json(model_dir: Path) -> dict[str, Any]:
|
||||
data = _read_bytes(model_dir / "config.json")
|
||||
if data is None:
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(data)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"{model_dir / 'config.json'}: the model config is unreadable ({error})") from error
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError(f"{model_dir / 'config.json'}: expected a JSON object") # noqa: TRY004 # the file's bytes are wrong, not the caller's argument
|
||||
return parsed
|
||||
|
||||
|
||||
def _model_hash(model_dir: Path) -> str:
|
||||
config = _read_bytes(model_dir / "config.json")
|
||||
if config is None:
|
||||
return UNKNOWN
|
||||
digest = hashlib.sha256(config)
|
||||
index = _read_bytes(model_dir / "model.safetensors.index.json")
|
||||
if index is not None:
|
||||
digest.update(index)
|
||||
return digest.hexdigest()
|
||||
for shard in sorted(model_dir.glob("*.safetensors")):
|
||||
digest.update(f"{shard.name}:{shard.stat().st_size}".encode())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _rope(config: dict[str, Any]) -> dict[str, Any]:
|
||||
rope: dict[str, Any] = {}
|
||||
for key in ("rope_parameters", "rope_scaling"):
|
||||
block = config.get(key)
|
||||
if isinstance(block, dict):
|
||||
rope.update(block)
|
||||
for key in ("partial_rotary_factor", "max_position_embeddings", "sliding_window"):
|
||||
value = config.get(key)
|
||||
if value is not None:
|
||||
rope[key] = value
|
||||
return rope
|
||||
|
||||
|
||||
def _chat_template_hash(model_dir: Path, config: dict[str, Any]) -> str:
|
||||
template = config.get("chat_template")
|
||||
if not isinstance(template, str):
|
||||
data = _read_bytes(model_dir / "chat_template.json")
|
||||
if data is None:
|
||||
return UNKNOWN
|
||||
try:
|
||||
parsed = json.loads(data)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"{model_dir / 'chat_template.json'}: unreadable ({error})") from error
|
||||
template = parsed.get("chat_template") if isinstance(parsed, dict) else None
|
||||
return _hex(template.encode("utf-8")) if isinstance(template, str) else UNKNOWN
|
||||
|
||||
|
||||
def adapter_hashes(adapter_root: Path) -> dict[str, str]:
|
||||
"""Hash every adapter weight under a directory, keyed by adapter kind."""
|
||||
found: dict[str, str] = {}
|
||||
if not adapter_root.is_dir():
|
||||
return found
|
||||
for weight in sorted(adapter_root.rglob("adapter_model.safetensors")):
|
||||
kind = weight.parent.name
|
||||
found[kind] = _hex(weight.read_bytes())
|
||||
return found
|
||||
|
||||
|
||||
def compute_fingerprint(settings: Settings, adapter_root: Path | None = None) -> KvFingerprint:
|
||||
"""Read what this machine can prove; unverifiable fields come back empty."""
|
||||
engine_version = UNKNOWN
|
||||
try:
|
||||
engine_version = metadata.version(
|
||||
"transformers" if settings.runtime_backend == "transformers" else "sglang"
|
||||
)
|
||||
except metadata.PackageNotFoundError:
|
||||
pass # an engine that is not installed here is a fact to report, not a crash
|
||||
model_dir = settings.model_dir
|
||||
config = _config_json(model_dir) if model_dir is not None else {}
|
||||
tokenizer = _read_bytes(model_dir / "tokenizer.json") if model_dir is not None else None
|
||||
if tokenizer is None and model_dir is not None:
|
||||
tokenizer = _read_bytes(model_dir / "tokenizer_config.json")
|
||||
return KvFingerprint(
|
||||
model_id=settings.model_id,
|
||||
model_hash=_model_hash(model_dir) if model_dir is not None else UNKNOWN,
|
||||
tokenizer_hash=_hex(tokenizer) if tokenizer is not None else UNKNOWN,
|
||||
chat_template_hash=_chat_template_hash(model_dir, config) if model_dir is not None else UNKNOWN,
|
||||
rope=_rope(config),
|
||||
context_tokens=settings.live_context_tokens,
|
||||
kv_dtype=settings.kv_cache_dtype,
|
||||
engine=settings.runtime_backend,
|
||||
engine_version=engine_version,
|
||||
adapter_hashes=adapter_hashes(adapter_root) if adapter_root is not None else {},
|
||||
)
|
||||
|
||||
|
||||
def unverifiable_fields(fingerprint: KvFingerprint) -> list[str]:
|
||||
"""Fields this machine cannot prove; a mount must not wave them through."""
|
||||
missing: list[str] = []
|
||||
for field in ("model_hash", "tokenizer_hash", "chat_template_hash", "engine_version"):
|
||||
if not getattr(fingerprint, field):
|
||||
missing.append(field)
|
||||
if not fingerprint.rope:
|
||||
missing.append("rope")
|
||||
return missing
|
||||
|
||||
|
||||
def fingerprint_diff(stored: KvFingerprint, live: KvFingerprint) -> list[str]:
|
||||
"""Named differences between a cartridge's header and this machine."""
|
||||
differences: list[str] = []
|
||||
if stored.format_semver != live.format_semver:
|
||||
differences.append(
|
||||
f"format_semver: cartridge {stored.format_semver} vs this build {live.format_semver}"
|
||||
)
|
||||
for field in (
|
||||
"model_id",
|
||||
"model_hash",
|
||||
"tokenizer_hash",
|
||||
"chat_template_hash",
|
||||
"context_tokens",
|
||||
"kv_dtype",
|
||||
"engine",
|
||||
"engine_version",
|
||||
):
|
||||
left, right = getattr(stored, field), getattr(live, field)
|
||||
if left != right:
|
||||
differences.append(f"{field}: cartridge {left!r} vs local {right!r}")
|
||||
if stored.rope != live.rope:
|
||||
keys = sorted(set(stored.rope) | set(live.rope))
|
||||
changes = [
|
||||
f"{key}: cartridge {stored.rope.get(key)!r} vs local {live.rope.get(key)!r}"
|
||||
for key in keys
|
||||
if stored.rope.get(key) != live.rope.get(key)
|
||||
]
|
||||
differences.append("rope: " + "; ".join(changes))
|
||||
if stored.adapter_hashes != live.adapter_hashes:
|
||||
kinds = sorted(set(stored.adapter_hashes) | set(live.adapter_hashes))
|
||||
changes = [
|
||||
f"{kind}: cartridge {_short(stored.adapter_hashes.get(kind))} vs local "
|
||||
f"{_short(live.adapter_hashes.get(kind))}"
|
||||
for kind in kinds
|
||||
if stored.adapter_hashes.get(kind) != live.adapter_hashes.get(kind)
|
||||
]
|
||||
differences.append("adapters: " + "; ".join(changes))
|
||||
return differences
|
||||
|
||||
|
||||
def _short(value: str | None) -> str:
|
||||
if not value:
|
||||
return "absent"
|
||||
return value[:12]
|
||||
@@ -14,7 +14,7 @@ import sqlite3
|
||||
|
||||
from .text_index import index_text
|
||||
|
||||
SCHEMA_VERSION = 3
|
||||
SCHEMA_VERSION = 4
|
||||
|
||||
ATOM_COLUMNS_V2 = """
|
||||
atom_id TEXT PRIMARY KEY,
|
||||
@@ -68,6 +68,20 @@ _BASE_TABLES = """
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Membership lives in the live store, not only in the copied parquet, otherwise
|
||||
-- "mount this cartridge" cannot scope a query against the atoms it holds.
|
||||
CREATE TABLE IF NOT EXISTS cartridge_atoms (
|
||||
cartridge_id TEXT NOT NULL REFERENCES cartridges(cartridge_id) ON DELETE CASCADE,
|
||||
atom_id TEXT NOT NULL REFERENCES atoms(atom_id) ON DELETE CASCADE,
|
||||
added_at TEXT NOT NULL,
|
||||
PRIMARY KEY (cartridge_id, atom_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mounts (
|
||||
cartridge_id TEXT PRIMARY KEY REFERENCES cartridges(cartridge_id) ON DELETE CASCADE,
|
||||
mounted_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cache_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts TEXT NOT NULL,
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
import base64
|
||||
from collections import defaultdict
|
||||
|
||||
from . import cartridge_store
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .embeddings import EmbeddingService
|
||||
@@ -44,11 +45,16 @@ class RetrievalService:
|
||||
if limit <= 0:
|
||||
raise ValueError(f"search needs a positive limit, got {limit}")
|
||||
offset = decode_cursor(cursor)
|
||||
criteria = (filters or SearchFilters()).model_dump(exclude_none=True)
|
||||
scope = criteria.get("cartridge_ids")
|
||||
if scope is None:
|
||||
scope = cartridge_store.mounted_ids(self.db) or None
|
||||
criteria["cartridge_ids"] = scope
|
||||
page = self.db.search_atoms(
|
||||
query,
|
||||
limit=limit * CANDIDATE_WINDOW,
|
||||
offset=offset,
|
||||
**(filters or SearchFilters()).model_dump(exclude_none=True),
|
||||
**criteria,
|
||||
)
|
||||
rankings: list[list[str]] = [[atom_id for atom_id, _ in page.rows]]
|
||||
scores: dict[str, dict[str, float]] = defaultdict(dict)
|
||||
@@ -56,6 +62,9 @@ class RetrievalService:
|
||||
scores[atom_id]["sparse_score"] = bm25
|
||||
if self.embeddings.enabled():
|
||||
dense = self._dense_search(query, limit * CANDIDATE_WINDOW)
|
||||
if scope:
|
||||
allowed = cartridge_store.atoms_of_cartridges(self.db, scope)
|
||||
dense = {atom_id: score for atom_id, score in dense.items() if atom_id in allowed}
|
||||
rankings.append(list(dense))
|
||||
for atom_id, cosine in dense.items():
|
||||
scores[atom_id]["dense_score"] = cosine
|
||||
|
||||
@@ -7,6 +7,8 @@ from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
from .kv_fingerprint import KvFingerprint
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
@@ -162,6 +164,7 @@ class SearchFilters(BaseModel):
|
||||
scenario: str | None = None
|
||||
source_id: str | None = None
|
||||
modality: Modality | None = None
|
||||
cartridge_ids: list[str] | None = None
|
||||
since: datetime | None = None
|
||||
until: datetime | None = None
|
||||
min_trust: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
@@ -256,6 +259,18 @@ class SystemStatus(BaseModel):
|
||||
profiles: dict[str, dict[str, Any]]
|
||||
data_home: str
|
||||
|
||||
class MountState(BaseModel):
|
||||
"""What is mounted now, and whether that narrows a search."""
|
||||
|
||||
mounted: list[str]
|
||||
scoped: bool
|
||||
atoms_in_scope: int
|
||||
|
||||
|
||||
class AppendRequest(BaseModel):
|
||||
atom_ids: list[str]
|
||||
|
||||
|
||||
class CartridgeManifest(BaseModel):
|
||||
cartridge_id: str
|
||||
name: str
|
||||
@@ -265,6 +280,7 @@ class CartridgeManifest(BaseModel):
|
||||
atom_count: int
|
||||
token_equivalent: int
|
||||
artifacts: dict[str, str]
|
||||
kv_fingerprint: KvFingerprint | None = None
|
||||
dna: str | None = None
|
||||
|
||||
class AdapterManifest(BaseModel):
|
||||
|
||||
Reference in new issue
Block a user