Give the write path a CLI and an MCP face

remember / supersede / forget now reach the one Python writer from a
terminal, from MCP, and from HTTP. The server, not the caller, decides
who owns a memory.

- cli: remember, supersede, forget. --holder falls back to
  settings.default_holder until the holder registry lands. Timestamps are
  parsed once with one clear message.
- ingestion: import pymupdf as fitz. The deprecated fitz import printed a
  warning to stdout, so every CLI command's JSON answer was unparseable.
- mcp: remember, supersede_memory and forget_memory forward to the Python
  writer over HTTP (ELY_EYE_API, default http://127.0.0.1:8765). SQLite
  stays read-only to the Go side, so the write rules live in one place.
- mcp reads follow schema v2: atoms.time became ingested_at, and reads go
  through v_atoms_current, so a superseded memory stops being findable.
  holder, origin, scenario and event_time now show up in fetch_atom and
  search_atoms.
- scripts/mcp_write_smoke.sh proves the loop end to end: remember, search,
  supersede, forget, then not found.

Evidence: 92 pytest passed; go build, go vet and go test clean; mypy clean
on the 10 touched source files with a fresh cache; ruff clean except a
pre-existing B008 at cli.py:292, identical at HEAD.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 15:09:17 -04:00
1 parent 181f6bf57e
commit 3d792417cf
20 files changed
+1425 -33

No files matched your search

+51
View File
@@ -35,6 +35,7 @@ from .evaluation_schemas import (
)
from .evidence_graph import EvidenceGraphService
from .ingestion import IngestionService
from .memory import MemoryService
from .profiles import runtime_profile_report
from .research_modules import ResearchModuleService
from .runtime import QwenRuntime, RuntimeUnavailableError
@@ -43,10 +44,16 @@ from .schemas import (
AdapterStatus,
AnswerResponse,
CartridgeAssetReport,
EvidenceAtom,
ForgetReport,
ForgetRequest,
MemoryOrigin,
MemoryStatus,
PathIngestRequest,
RememberRequest,
RuntimeProfile,
RuntimeProfileReport,
SupersedeRequest,
SystemStatus,
UploadIngestRequest,
)
@@ -183,6 +190,50 @@ def create_app() -> FastAPI:
runtime_backend=generation.backend,
)
@app.post("/api/memory", response_model=EvidenceAtom)
def remember(
request: RememberRequest, settings: Annotated[Settings, Depends(settings_dep)]
) -> EvidenceAtom:
try:
return MemoryService(settings).remember(
request.text,
holder_id=settings.default_holder,
origin=MemoryOrigin.self_,
scenario=request.scenario,
event_time=request.event_time,
event_time_precision=request.event_time_precision,
event_time_source=request.event_time_source,
retention_until=request.retention_until,
)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@app.post("/api/memory/supersede", response_model=EvidenceAtom)
def supersede(
request: SupersedeRequest, settings: Annotated[Settings, Depends(settings_dep)]
) -> EvidenceAtom:
try:
return MemoryService(settings).supersede(
request.atom_id,
text=request.text,
holder_id=settings.default_holder,
scenario=request.scenario,
event_time=request.event_time,
event_time_precision=request.event_time_precision,
event_time_source=request.event_time_source,
)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@app.post("/api/memory/forget", response_model=ForgetReport)
def forget(request: ForgetRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> ForgetReport:
try:
return MemoryService(settings).forget(
atom_id=request.atom_id, source_id=request.source_id, reason=request.reason
)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@app.get("/api/cartridges")
def cartridges(settings: Annotated[Settings, Depends(settings_dep)]):
return CartridgeService(settings).list_manifests()
+79 -12
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from datetime import datetime
from pathlib import Path
import typer
@@ -20,10 +21,11 @@ from .db import Database
from .embeddings import EmbeddingService
from .evals import HashHopEvaluator, PRDProofSuite
from .ingestion import IngestionService
from .memory import MemoryService
from .profiles import runtime_profile_report
from .research_modules import ResearchModuleService
from .runtime import QwenRuntime
from .schemas import TrainingAdapterKind
from .schemas import MemoryOrigin, TimePrecision, TrainingAdapterKind
from .training import TrainingService
app = typer.Typer(no_args_is_help=True)
@@ -46,17 +48,19 @@ def status() -> None:
runtime = QwenRuntime(settings).status()
db = Database(settings)
typer.echo(
{
"runtime": runtime.model_dump(mode="json"),
"atoms": db.atom_count(),
"token_equivalent": db.token_equivalent_sum(),
"library_token_equivalent": max(
db.token_equivalent_sum(),
db.cartridge_token_equivalent_sum(),
),
"cartridges": db.cartridge_count(),
"data_home": str(settings.home),
}
json_dump(
{
"runtime": runtime.model_dump(mode="json"),
"atoms": db.atom_count(),
"token_equivalent": db.token_equivalent_sum(),
"library_token_equivalent": max(
db.token_equivalent_sum(),
db.cartridge_token_equivalent_sum(),
),
"cartridges": db.cartridge_count(),
"data_home": str(settings.home),
}
)
)
@@ -66,6 +70,69 @@ def ingest(path: Path, cartridge_name: str | None = None) -> None:
typer.echo(result.model_dump_json(indent=2))
def _moment(value: str, option: str) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value)
except ValueError as error:
raise ValueError(f"{option} {value!r} is not an ISO timestamp: {error}") from error
@app.command()
def remember(
text: str,
holder: str = typer.Option("", "--holder", help="who owns this memory"),
origin: MemoryOrigin = MemoryOrigin.self_,
scenario: str = "",
event_time: str = typer.Option("", "--event-time", help="when it was true, ISO with offset"),
precision: TimePrecision = TimePrecision.unknown,
retention_until: str = typer.Option("", "--retention-until"),
) -> None:
settings = get_settings()
atom = MemoryService(settings).remember(
text,
holder_id=holder or settings.default_holder,
origin=origin,
scenario=scenario,
event_time=_moment(event_time, "--event-time"),
event_time_precision=precision,
retention_until=_moment(retention_until, "--retention-until"),
)
typer.echo(atom.model_dump_json(indent=2))
@app.command()
def supersede(
atom_id: str,
text: str,
holder: str = typer.Option("", "--holder"),
scenario: str = "",
event_time: str = typer.Option("", "--event-time"),
precision: TimePrecision = TimePrecision.unknown,
) -> None:
settings = get_settings()
atom = MemoryService(settings).supersede(
atom_id,
text=text,
holder_id=holder or settings.default_holder,
scenario=scenario,
event_time=_moment(event_time, "--event-time"),
event_time_precision=precision,
)
typer.echo(atom.model_dump_json(indent=2))
@app.command()
def forget(atom_id: str = "", source_id: str = "", reason: str = "") -> None:
report = MemoryService(get_settings()).forget(
atom_id=atom_id or None,
source_id=source_id or None,
reason=reason,
)
typer.echo(report.model_dump_json(indent=2))
@app.command("build-dense-index")
def build_dense_index() -> None:
index = EmbeddingService(get_settings()).build_index()
+1
View File
@@ -12,6 +12,7 @@ class Settings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="ELY_EYE_", env_file=".env", extra="ignore")
home: Path = Field(default_factory=lambda: Path.cwd() / ".ely_eye")
default_holder: str = "local"
model_id: str = "Qwen/Qwen3.5-9B"
planner_model_id: str = "Qwen/Qwen3.5-0.8B"
embedding_model_id: str = "Qwen/Qwen3-VL-Embedding-8B"
+60
View File
@@ -221,6 +221,66 @@ class Database:
).fetchall()
return [(row["atom_id"], float(-row["score"])) for row in rows]
def source_id_of_atom(self, atom_id: str) -> str | None:
with self.connect() as connection:
row = connection.execute("SELECT source_id FROM atoms WHERE atom_id = ?", (atom_id,)).fetchone()
return None if row is None else str(row["source_id"])
def atoms_of_source(self, source_id: str) -> list[EvidenceAtom]:
with self.connect() as connection:
rows = connection.execute(
"SELECT * FROM atoms WHERE source_id = ? ORDER BY atom_id", (source_id,)
).fetchall()
return [self._row_to_atom(row) for row in rows]
def close_atoms(self, atom_ids: list[str], closed_at: datetime) -> int:
"""Stop treating these memories as current; the rows stay for time travel."""
if not atom_ids:
return 0
with self.connect() as connection:
cursor = connection.execute(
"UPDATE atoms SET valid_to = ? "
"WHERE valid_to IS NULL AND atom_id IN (SELECT value FROM json_each(?))",
(closed_at.isoformat(), _json(atom_ids)),
)
return int(cursor.rowcount)
def delete_atoms(self, atom_ids: list[str]) -> int:
if not atom_ids:
return 0
with self.connect() as connection:
connection.execute(
"DELETE FROM atom_fts WHERE atom_id IN (SELECT value FROM json_each(?))", (_json(atom_ids),)
)
cursor = connection.execute(
"DELETE FROM atoms WHERE atom_id IN (SELECT value FROM json_each(?))", (_json(atom_ids),)
)
return int(cursor.rowcount)
def delete_edges_touching(self, atom_ids: list[str]) -> int:
if not atom_ids:
return 0
with self.connect() as connection:
cursor = connection.execute(
"DELETE FROM evidence_edges "
"WHERE from_atom_id IN (SELECT value FROM json_each(?)) "
"OR to_atom_id IN (SELECT value FROM json_each(?))",
(_json(atom_ids), _json(atom_ids)),
)
return int(cursor.rowcount)
def clear_cache_events(self, reason: str) -> int:
"""Empty the cache and leave one trace row proving why it is empty."""
with self.connect() as connection:
cursor = connection.execute("DELETE FROM cache_events")
cleared = int(cursor.rowcount)
connection.execute(
"INSERT INTO cache_events(ts, layer, cache_key, event, bytes_estimate, metadata_json) "
"VALUES (?, 'memory', ?, 'invalidated', 0, ?)",
(utc_now().isoformat(), reason, _json({"entries_cleared": cleared})),
)
return cleared
def atom_count(self) -> int:
with self.connect() as connection:
return int(connection.execute("SELECT COUNT(*) FROM atoms").fetchone()[0])
+11 -5
View File
@@ -7,12 +7,11 @@ import re
import subprocess
import tempfile
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import fitz
import cv2
import pymupdf as fitz
from PIL import Image
from .config import Settings, get_settings
@@ -26,8 +25,14 @@ from .schemas import (
SourceRecord,
TrustScores,
)
from .storage import ObjectStore, guess_mime, safe_source_id, sha256_bytes, sha256_file, token_equivalent
from .storage import (
ObjectStore,
guess_mime,
safe_source_id,
sha256_bytes,
sha256_file,
token_equivalent,
)
SUPPORTED_TEXT_EXTENSIONS = {
".txt",
@@ -440,7 +445,8 @@ def code_context_entry(root: Path, path: Path) -> dict[str, Any]:
def repository_call_graph(code_entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
edges: list[dict[str, Any]] = []
for entry in code_entries:
symbols = entry.get("symbols") if isinstance(entry.get("symbols"), dict) else {}
declared = entry.get("symbols")
symbols = declared if isinstance(declared, dict) else {}
for call in symbols.get("calls", []):
if not isinstance(call, dict):
continue
+219
View File
@@ -0,0 +1,219 @@
"""The single writer for memory: remember, supersede, forget.
Ingestion turns artefacts into atoms. This module turns statements into memories, and it
is the only place a memory is created or destroyed, so provenance is stamped here and no
caller can claim to be somebody else.
"""
from __future__ import annotations
import hashlib
import json
from datetime import datetime
from pathlib import Path
from typing import Any
from .config import Settings, get_settings
from .db import Database
from .schemas import (
EvidenceAtom,
ForgetReport,
MemoryOrigin,
Modality,
SourceRecord,
TimePrecision,
TrustScores,
Visibility,
utc_now,
)
from .storage import token_equivalent
# Trust scores describe extraction fidelity, not truthfulness: a remembered sentence is
# stored verbatim, so the parser is fully trusted and origin carries the epistemic weight.
MEMORY_TRUST = TrustScores(parser=1.0, ocr=0.0, model=0.0)
class MemoryService:
def __init__(self, settings: Settings | None = None, database: Database | None = None) -> None:
self.settings = settings or get_settings()
self.db = database or Database(self.settings)
def remember(
self,
text: str,
*,
holder_id: str,
origin: MemoryOrigin,
scenario: str = "",
event_time: datetime | None = None,
event_time_precision: TimePrecision = TimePrecision.unknown,
event_time_source: str = "",
visibility: Visibility | None = None,
supersedes: list[str] | None = None,
valid_from: datetime | None = None,
retention_until: datetime | None = None,
) -> EvidenceAtom:
"""Store one statement as a memory owned by a holder.
Repeating the same sentence as the same holder updates the memory instead of
duplicating it; two holders holding the same sentence keep two memories.
"""
statement = text.strip()
if not statement:
raise ValueError("remember needs text; an empty memory will be found by nobody")
if not holder_id.strip():
raise ValueError("remember needs a holder; a memory nobody owns cannot be referred to")
if event_time is not None and event_time_precision is TimePrecision.unknown:
raise ValueError(
f"event_time {event_time.isoformat()} needs its precision; "
"a day stored as an instant will be read as a moment"
)
moment = utc_now()
digest = hashlib.sha256(f"{holder_id}\n{statement}".encode()).hexdigest()
source_id = f"mem-{digest[:16]}"
self.db.upsert_sources(
[
SourceRecord(
source_id=source_id,
path=f"memory://{holder_id}/{scenario or 'general'}",
kind="memory",
mime="text/plain",
sha256=hashlib.sha256(statement.encode("utf-8")).hexdigest(),
size_bytes=len(statement.encode("utf-8")),
created_at=moment,
metadata={"holder_id": holder_id, "origin": origin.value},
)
]
)
atom = EvidenceAtom(
atom_id=f"{source_id}:text",
modality=Modality.text,
source_id=source_id,
source=f"memory://{holder_id}/{scenario or 'general'}",
ingested_at=moment,
text=statement,
trust=MEMORY_TRUST,
token_equivalent=token_equivalent(statement),
holder_id=holder_id,
origin=origin,
scenario=scenario,
event_time=event_time,
event_time_precision=event_time_precision,
event_time_source=event_time_source,
supersedes=list(supersedes or []),
valid_from=valid_from,
visibility=visibility or Visibility(),
retention_until=retention_until,
)
self.db.upsert_atoms([atom])
return atom
def supersede(
self,
atom_id: str,
*,
text: str,
holder_id: str,
origin: MemoryOrigin = MemoryOrigin.self_,
scenario: str = "",
event_time: datetime | None = None,
event_time_precision: TimePrecision = TimePrecision.unknown,
event_time_source: str = "",
) -> EvidenceAtom:
"""Close an old memory and record the new one as its replacement.
Both rows stay: the new atom answers what is true now, the closed one answers what
was true when asked, and a system that forgets the second cannot explain itself.
"""
previous = self.db.get_atoms([atom_id])
if not previous:
raise ValueError(f"cannot supersede {atom_id}: no such atom")
if previous[0].valid_to is not None:
raise ValueError(
f"{atom_id} was already closed at {previous[0].valid_to.isoformat()}; "
"superseding a closed memory hides which replacement is current"
)
# A correction takes effect when it is asserted: the old memory stops being current
# and the new one starts there, so the pair never claims to be true at the same time.
moment = event_time or utc_now()
replacement = self.remember(
text,
holder_id=holder_id,
origin=origin,
scenario=scenario or previous[0].scenario,
event_time=event_time,
event_time_precision=event_time_precision,
event_time_source=event_time_source,
supersedes=[atom_id],
valid_from=moment,
)
self.db.close_atoms([atom_id], moment)
return replacement
def forget(
self,
*,
atom_id: str | None = None,
source_id: str | None = None,
reason: str = "",
) -> ForgetReport:
"""Delete a memory and everything derived from it.
The cascade is the whole point: an answer cached from a forgotten memory is still
on screen, and a vector index that still carries it will keep surfacing it.
"""
if (atom_id is None) == (source_id is None):
raise ValueError("forget takes exactly one of atom_id or source_id")
target_source = source_id or self.db.source_id_of_atom(str(atom_id))
if target_source is None:
raise ValueError(f"nothing to forget: no atom {atom_id}")
doomed = self.db.atoms_of_source(target_source)
if not doomed:
raise ValueError(f"nothing to forget: source {target_source} holds no atom")
doomed_ids = [atom.atom_id for atom in doomed]
edges_deleted = self.db.delete_edges_touching(doomed_ids)
atoms_deleted = self.db.delete_atoms(doomed_ids)
cache_cleared = self.db.clear_cache_events(f"forget:{target_source}")
vectors_dropped = self._drop_vector_index()
return ForgetReport(
source_id=target_source,
atom_ids=doomed_ids,
atoms_deleted=atoms_deleted,
edges_deleted=edges_deleted,
cache_entries_cleared=cache_cleared,
vector_index_dropped=vectors_dropped,
affected_cartridges=self._cartridges_holding(target_source),
reason=reason,
)
def _drop_vector_index(self) -> bool:
"""The dense index is derived data; a stale one keeps answering with deleted text."""
dropped = False
for path in (self.settings.cache_dir / "dense_index.npy", self.settings.cache_dir / "dense_index_atoms.json"):
if path.exists():
path.unlink()
dropped = True
return dropped
def _cartridges_holding(self, source_id: str) -> list[dict[str, Any]]:
"""Name the cartridges that exported this source, so they can be rebuilt.
An exported cartridge is an immutable artefact on disk; rewriting it here would
silently change a DNA hash somebody already quoted.
"""
affected: list[dict[str, Any]] = []
for cartridge in self.db.list_cartridges():
sources_path = Path(str(cartridge["root_path"])) / "sources.json"
try:
payload = json.loads(sources_path.read_text(encoding="utf-8"))
except FileNotFoundError:
affected.append({"cartridge_id": cartridge["cartridge_id"], "contains_source": "unknown"})
continue
except (OSError, json.JSONDecodeError) as error:
raise ValueError(f"{sources_path}: the cartridge source list is unreadable ({error})") from error
held = any(str(row.get("source_id")) == source_id for row in payload)
if held:
affected.append({"cartridge_id": cartridge["cartridge_id"], "contains_source": True})
return affected
+47 -1
View File
@@ -5,7 +5,7 @@ from enum import StrEnum
from pathlib import Path
from typing import Any, Literal
from pydantic import BaseModel, Field, model_validator
from pydantic import BaseModel, Field, field_validator, model_validator
def utc_now() -> datetime:
@@ -116,6 +116,16 @@ class EvidenceAtom(BaseModel):
raise ValueError(f"{self.atom_id}: an atom cannot supersede itself")
return self
@field_validator("ingested_at", "event_time", "event_time_end", "valid_from", "valid_to", "retention_until")
@classmethod
def _times_are_aware(cls, value: datetime | None) -> datetime | None:
if value is not None and value.tzinfo is None:
raise ValueError(
"memory timestamps need an explicit offset, e.g. 2026-05-01T12:00:00+08:00; "
"a naive time cannot be compared with one that has an offset"
)
return value
class SourceRecord(BaseModel):
source_id: str
path: str
@@ -378,6 +388,42 @@ class RuntimeProfileReport(BaseModel):
profiles: list[RuntimeLaunchProfile]
validations: list[RuntimeProfileValidation] = Field(default_factory=list)
class ForgetReport(BaseModel):
"""What a forget actually removed, including the derived copies that have to go with it."""
source_id: str
atom_ids: list[str]
atoms_deleted: int
edges_deleted: int
cache_entries_cleared: int
vector_index_dropped: bool
affected_cartridges: list[dict[str, Any]]
reason: str
class RememberRequest(BaseModel):
"""A memory as a caller states it. Holder, origin and ingestion time are stamped by the
server, so they are absent here on purpose: a client cannot claim to be somebody else.
"""
text: str
scenario: str = ""
event_time: datetime | None = None
event_time_precision: TimePrecision = TimePrecision.unknown
event_time_source: str = ""
retention_until: datetime | None = None
class SupersedeRequest(RememberRequest):
atom_id: str
class ForgetRequest(BaseModel):
atom_id: str | None = None
source_id: str | None = None
reason: str = ""
class UploadIngestRequest(BaseModel):
cartridge_name: str | None = None
dense_embeddings: bool | None = None