Require a token for writes and quarantine what nobody vouches for

Sixteen mutating routes now refuse a caller who cannot present the token the
server started with; queries stay open, because a question is not a write. The
environment variable is ELY_EYE_AUTH_TOKEN, following the repo's env_prefix
convention — the shorter name I reached for first returned 403 where the smoke
expected 401, and the server printed the reason.

Words from a stranger are quarantined on the way in: an HTTP upload is
third_party by definition, and a sentence carrying an instruction-shaped phrase
is quarantined whoever wrote it. A quarantined atom stays findable and stays out
of the compiled context until a second, live, unquarantined source agrees with
its rarest words. Corroboration is computed at read time because it is a
property of the corpus now, not of the moment a memory was written; it is
term-level, and the docstring says so.

forget now erases the source row and the compiled contexts beside the atoms, so
a forgotten memory leaves nothing that can still speak. The semantic cache key
carries a store revision, which fixes a real staleness: a context compiled
before new evidence arrived kept its old quarantine decision forever.

bench.audit --product still pointed at evals.py, which the p1-6 split moved into
backend/src/ely_eye/proofs/. It failed fast, but the anti-self-justification
rule had quietly stopped auditing the product; it now walks the package, and a
test pins that the package exists and holds no re-roll.

Gates: pytest 306 passed; ruff clean; mypy cold cache 24 errors in 9 files, the
recorded baseline, net zero; pyright 0 errors on the 23 backend files and the 2
bench files; go build/vet/test and gofmt clean; npm run build 221.39 kB with
eslint exit 0; bench/filesize_check clean; bench.audit --product clean over 31
harness files; scripts/mcp_write_smoke.sh PASS with the gate live across two
servers and the MCP child.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-08 23:05:15 -04:00
1 parent 8b974e6626
commit 9145a23f18
32 files changed
+750 -58

No files matched your search

+28 -7
View File
@@ -13,6 +13,7 @@ from pydantic import BaseModel
from . import __version__
from .adapters import plan as adapter_plan
from .api_auth import require_write
from .api_cartridges import register as register_cartridge_routes
from .api_holders import register as register_holder_routes
from .browse import browse, picture
@@ -76,9 +77,10 @@ def settings_dep() -> Settings:
def create_app() -> FastAPI:
app = FastAPI(title="Ely-Eye", version=__version__)
app.state.write_settings = settings_dep
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"],
allow_origins=settings_dep().allowed_origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
@@ -116,6 +118,7 @@ def create_app() -> FastAPI:
def ingest_path(
request: PathIngestRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
):
try:
return IngestionService(settings).ingest_path(request.path, request.cartridge_name)
@@ -126,6 +129,7 @@ def create_app() -> FastAPI:
def ingest_upload(
metadata: Annotated[UploadIngestRequest, Depends()],
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
files: Annotated[list[UploadFile], File()],
):
upload_root = settings.data_dir / "uploads"
@@ -144,7 +148,7 @@ def create_app() -> FastAPI:
service = IngestionService(settings)
aggregate = None
for path in saved_paths:
result = service.ingest_path(path, None)
result = service.ingest_path(path, None, origin=MemoryOrigin.third_party)
if aggregate is None:
aggregate = result
else:
@@ -162,7 +166,11 @@ def create_app() -> FastAPI:
return aggregate
@app.post("/api/chat", response_model=AnswerResponse)
def chat(request: ChatRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> AnswerResponse:
def chat(
request: ChatRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> AnswerResponse:
db = Database(settings)
compiler = ContextCompiler(settings, db)
context = compiler.compile(request.question, request.profile)
@@ -200,7 +208,9 @@ def create_app() -> FastAPI:
@app.post("/api/memory", response_model=EvidenceAtom)
def remember(
request: RememberRequest, settings: Annotated[Settings, Depends(settings_dep)]
request: RememberRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> EvidenceAtom:
try:
return MemoryService(settings).remember(
@@ -218,7 +228,9 @@ def create_app() -> FastAPI:
@app.post("/api/memory/supersede", response_model=EvidenceAtom)
def supersede(
request: SupersedeRequest, settings: Annotated[Settings, Depends(settings_dep)]
request: SupersedeRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> EvidenceAtom:
try:
return MemoryService(settings).supersede(
@@ -234,7 +246,11 @@ def create_app() -> FastAPI:
raise HTTPException(status_code=422, detail=str(exc)) from exc
@app.post("/api/memory/forget", response_model=ForgetReport)
def forget(request: ForgetRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> ForgetReport:
def forget(
request: ForgetRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> ForgetReport:
try:
return MemoryService(settings).forget(
atom_id=request.atom_id, source_id=request.source_id, reason=request.reason
@@ -288,7 +304,11 @@ def create_app() -> FastAPI:
return adapter_plan(settings, Database(settings))
@app.post("/api/hashhop", response_model=HashHopProof)
def hashhop(request: HashHopRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> HashHopProof:
def hashhop(
request: HashHopRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> HashHopProof:
evaluator = HashHopEvaluator(settings)
if request.kind == "text":
return evaluator.generate_text_proof(request.hops, request.token_equivalent)
@@ -300,6 +320,7 @@ def create_app() -> FastAPI:
def proof_suite(
request: ProofSuiteRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> ProofSuiteReport:
try:
return PRDProofSuite(settings).run(request.cartridge_id)
+42
View File
@@ -0,0 +1,42 @@
"""Writes need a token; queries do not.
Anything that can change stored state — a memory, an ingested source, a mounted cartridge, a
proof run — is refused unless the caller presents the token the server was started with. A
browser must also come from an allowed origin. The CLI and the MCP server reach the same
writer through the same rule, so no face of the product gets a cheaper path to the store.
"""
from __future__ import annotations
import secrets
from typing import TYPE_CHECKING
from fastapi import HTTPException, Request
if TYPE_CHECKING:
from .config import Settings
def require_write(request: Request) -> None:
"""Gate a mutating request. The app carries the settings source, so this stays a plain global."""
settings: Settings = request.app.state.write_settings()
origin = request.headers.get("origin")
if origin is not None and origin not in settings.allowed_origins:
raise HTTPException(
status_code=403,
detail=f"origin {origin} may not write; allowed origins: {', '.join(settings.allowed_origins)}",
)
configured = settings.auth_token
if not configured:
raise HTTPException(
status_code=403,
detail="this server accepts no writes: start it with ELY_EYE_AUTH_TOKEN set",
)
header = request.headers.get("authorization", "")
presented = header.removeprefix("Bearer ").strip() if header.startswith("Bearer ") else ""
if not presented or not secrets.compare_digest(presented, configured):
raise HTTPException(
status_code=401,
detail="write needs the token: send Authorization: Bearer $ELY_EYE_AUTH_TOKEN",
)
+10
View File
@@ -7,6 +7,7 @@ from typing import Annotated
from fastapi import Depends, FastAPI, HTTPException
from .api_auth import require_write
from .cartridge import CartridgeService
from .cartridge_assets import CartridgeAssetService
from .config import Settings
@@ -17,6 +18,9 @@ from .schemas import AppendRequest, CartridgeAssetReport, CartridgeManifest, Mou
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
"""Mounting decides what a search may see; the KV fingerprint decides if it may mount here."""
app.state.write_settings = settings_dep
@app.get("/api/cartridges")
def cartridges(settings: Annotated[Settings, Depends(settings_dep)]):
return CartridgeService(settings).list_manifests()
@@ -29,6 +33,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
def mount_cartridge(
cartridge_id: str,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> CartridgeManifest:
try:
return CartridgeService(settings).mount(cartridge_id)
@@ -39,6 +44,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
def switch_cartridge(
cartridge_id: str,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> CartridgeManifest:
try:
return CartridgeService(settings).switch(cartridge_id)
@@ -49,6 +55,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
def unmount_cartridge(
cartridge_id: str,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> MountState:
try:
return CartridgeService(settings).unmount(cartridge_id)
@@ -60,6 +67,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
cartridge_id: str,
request: AppendRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> CartridgeManifest:
try:
return CartridgeService(settings).append(cartridge_id, request.atom_ids)
@@ -70,6 +78,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
def finalize_cartridge(
cartridge_id: str,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> CartridgeAssetReport:
try:
return CartridgeAssetService(settings).finalize(cartridge_id)
@@ -80,6 +89,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
def refresh_cartridge(
cartridge_id: str,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
):
db = Database(settings)
try:
+6
View File
@@ -10,6 +10,7 @@ from typing import Annotated
from fastapi import Depends, FastAPI, HTTPException
from .api_auth import require_write
from .config import Settings
from .db import Database
from .holder_store import HolderStore
@@ -28,9 +29,12 @@ from .transactive_schemas import (
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
"""Register the transactive routes. The server owns the timestamps, the caller does not."""
app.state.write_settings = settings_dep
def store(settings: Settings) -> HolderStore:
return HolderStore(Database(settings))
@app.get("/api/holders", response_model=list[Holder])
def holders(settings: Annotated[Settings, Depends(settings_dep)]) -> list[Holder]:
return store(settings).list_holders()
@@ -40,6 +44,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
holder_id: str,
request: HolderUpsertRequest,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> Holder:
now = utc_now()
entry = Holder(
@@ -67,6 +72,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
def delete_holder(
holder_id: str,
settings: Annotated[Settings, Depends(settings_dep)],
_: Annotated[None, Depends(require_write)],
) -> Holder:
registry = store(settings)
entry = registry.get(holder_id)
+13 -2
View File
@@ -17,8 +17,9 @@ class CacheFabric:
self.settings = settings or get_settings()
self.db = database or Database(self.settings)
def semantic_key(self, question: str, profile: str) -> str:
return hashlib.sha256(f"compiler-v2\n{profile}\n{question}".encode()).hexdigest()
def semantic_key(self, question: str, profile: str, store_revision: int) -> str:
"""A compiled context is only true for the corpus it was compiled from, so the corpus is part of its name."""
return hashlib.sha256(f"compiler-v2\n{profile}\n{store_revision}\n{question}".encode()).hexdigest()
def record_hit(self, layer: str, cache_key: str, bytes_estimate: int = 0, metadata: dict[str, Any] | None = None) -> None:
self.db.log_cache_event(layer, cache_key, "hit", bytes_estimate, metadata)
@@ -43,6 +44,16 @@ class CacheFabric:
except (OSError, json.JSONDecodeError) as error:
raise ValueError(f"{path}: the semantic cache entry is unreadable ({error})") from error
def drop_semantic_cache(self) -> int:
"""Delete every compiled context: each one may quote a memory that no longer exists."""
directory = self.settings.cache_dir / "semantic"
if not directory.exists():
return 0
entries = list(directory.glob("*.json"))
for path in entries:
path.unlink()
return len(entries)
def statuses(self) -> list[CacheLayerStatus]:
rows = self.db.cache_layer_rows()
statuses: list[CacheLayerStatus] = []
+5
View File
@@ -46,6 +46,11 @@ register_maintenance_commands(app, lambda: get_settings())
@app.command()
def serve(host: str = "127.0.0.1", port: int = 8765) -> None:
settings = get_settings()
if settings.auth_token:
typer.echo(f"writes need Authorization: Bearer $ELY_EYE_AUTH_TOKEN; {len(settings.allowed_origins)} allowed origin(s)")
else:
typer.echo("writes are closed on this server: start it with ELY_EYE_AUTH_TOKEN set")
uvicorn.run(create_app(), host=host, port=port)
+14 -2
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
from collections.abc import Iterable
from . import trust
from .cache_fabric import SEMANTIC_LAYER, CacheFabric
from .config import Settings, get_settings
from .db import Database
@@ -25,7 +26,7 @@ class ContextCompiler:
self.cache = CacheFabric(self.settings, self.db)
def compile(self, question: str, profile: RuntimeProfile) -> CompiledContext:
cache_key = self.cache.semantic_key(question, profile.value)
cache_key = self.cache.semantic_key(question, profile.value, self.db.revision())
cached = self.cache.read_semantic_cache(cache_key)
if cached is not None:
compiled = CompiledContext.model_validate(cached)
@@ -40,7 +41,8 @@ class ContextCompiler:
self.cache.record_miss(SEMANTIC_LAYER, cache_key)
plan = self.plan(question, profile)
hits = self.retrieval.search(plan.retrieval_query, plan.evidence_budget_atoms).hits
packed_text, token_total = self._pack_hits(hits, plan.token_budget, plan.retrieval_query)
citable, held = self._split_citable(hits)
packed_text, token_total = self._pack_hits(citable, plan.token_budget, plan.retrieval_query)
trace_id = hashlib.sha256(f"{cache_key}:{token_total}".encode()).hexdigest()[:16]
compiled = CompiledContext(
plan=plan,
@@ -48,6 +50,7 @@ class ContextCompiler:
packed_text=packed_text,
token_equivalent=token_total,
cache_trace_id=trace_id,
quarantined_excluded=[hit.atom.atom_id for hit in held],
)
self.cache.write_semantic_cache(cache_key, compiled.model_dump(mode="json"))
return compiled
@@ -71,6 +74,15 @@ class ContextCompiler:
],
)
def _split_citable(self, hits: list[RetrievalHit]) -> tuple[list[RetrievalHit], list[RetrievalHit]]:
"""A quarantined atom reaches the prompt only when a trusted source says the same thing."""
citable: list[RetrievalHit] = []
held: list[RetrievalHit] = []
for hit in hits:
(citable if trust.is_citable(hit.atom, self.db) else held).append(hit)
return citable, held
def _compression_strategy(self, profile: RuntimeProfile, modalities: Iterable[Modality]) -> str:
has_visual = any(modality in {Modality.image, Modality.pdf_page, Modality.video_frame, Modality.ui_screenshot} for modality in modalities)
if profile == RuntimeProfile.extreme_context:
+4
View File
@@ -13,6 +13,10 @@ class Settings(BaseSettings):
home: Path = Field(default_factory=lambda: Path.cwd() / ".ely_eye")
default_holder: str = "local"
auth_token: str | None = None
allowed_origins: list[str] = Field(
default_factory=lambda: ["http://localhost:5173", "http://127.0.0.1:5173"]
)
model_id: str = "Qwen/Qwen3.5-9B"
model_dir: Path | None = None
kv_cache_dtype: str = "auto"
+25 -3
View File
@@ -81,9 +81,10 @@ class Database:
embedding_refs_json, layout_json, relations_json, trust_json,
token_equivalent, metadata_json, holder_id, origin, scenario,
event_time, event_time_end, event_time_precision, event_time_source,
valid_from, valid_to, supersedes_json, visibility_json, retention_until
valid_from, valid_to, supersedes_json, visibility_json, retention_until,
quarantined
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(atom_id) DO UPDATE SET
modality=excluded.modality,
source=excluded.source,
@@ -107,7 +108,8 @@ class Database:
valid_to=excluded.valid_to,
supersedes_json=excluded.supersedes_json,
visibility_json=excluded.visibility_json,
retention_until=excluded.retention_until
retention_until=excluded.retention_until,
quarantined=excluded.quarantined
""",
[atom_params(atom) for atom in atoms],
)
@@ -118,6 +120,7 @@ class Database:
)
# New text changes which words are common, so every remembered count is void.
connection.execute("DELETE FROM term_df")
self._bump_revision(connection)
def list_atoms(self, limit: int | None = None) -> list[EvidenceAtom]:
with self.connect() as connection:
@@ -188,6 +191,7 @@ class Database:
"WHERE valid_to IS NULL AND atom_id IN (SELECT value FROM json_each(?))",
(closed_at.isoformat(), to_json(atom_ids)),
)
self._bump_revision(connection)
return int(cursor.rowcount)
def delete_atoms(self, atom_ids: list[str]) -> int:
@@ -201,6 +205,13 @@ class Database:
"DELETE FROM atoms WHERE atom_id IN (SELECT value FROM json_each(?))", (to_json(atom_ids),)
)
connection.execute("DELETE FROM term_df")
self._bump_revision(connection)
return int(cursor.rowcount)
def delete_source(self, source_id: str) -> int:
"""Erase where a memory came from, so a forget leaves no provenance row behind."""
with self.connect() as connection:
cursor = connection.execute("DELETE FROM sources WHERE source_id = ?", (source_id,))
return int(cursor.rowcount)
def delete_edges_touching(self, atom_ids: list[str]) -> int:
@@ -231,6 +242,17 @@ class Database:
with self.connect() as connection:
return int(connection.execute("SELECT COUNT(*) FROM atoms").fetchone()[0])
def revision(self) -> int:
"""What a compiled context was compiled against; see store_meta."""
with self.connect() as connection:
row = connection.execute("SELECT value FROM store_meta WHERE key = 'revision'").fetchone()
return int(row["value"]) if row is not None else 0
def _bump_revision(self, connection: sqlite3.Connection) -> None:
"""A write changes what a compiled context may contain, so the semantic cache must notice."""
connection.execute("INSERT OR IGNORE INTO store_meta(key, value) VALUES ('revision', '0')")
connection.execute("UPDATE store_meta SET value = CAST(value AS INTEGER) + 1 WHERE key = 'revision'")
def token_equivalent_sum(self) -> int:
with self.connect() as connection:
value = connection.execute("SELECT COALESCE(SUM(token_equivalent), 0) FROM atoms").fetchone()[0]
+10 -1
View File
@@ -6,6 +6,7 @@ import os
from pathlib import Path
from typing import Any
from . import trust
from .asr import SpeechTranscriber
from .config import Settings, get_settings
from .db import Database
@@ -25,6 +26,7 @@ from .ingestion_repository import RepositoryIngest, extract_symbols
from .schemas import (
EvidenceAtom,
IngestResponse,
MemoryOrigin,
Modality,
SkippedFile,
SourceRecord,
@@ -55,7 +57,13 @@ class IngestionService(MediaIngest, RepositoryIngest):
self.transcriber = SpeechTranscriber(self.settings)
self.vectors = vectors or VectorIndex(self.settings, self.db)
def ingest_path(self, path: Path, cartridge_name: str | None = None) -> IngestResponse:
def ingest_path(
self,
path: Path,
cartridge_name: str | None = None,
*,
origin: MemoryOrigin = MemoryOrigin.artifact,
) -> IngestResponse:
path = path.expanduser().resolve()
if not path.exists():
raise FileNotFoundError(path)
@@ -79,6 +87,7 @@ class IngestionService(MediaIngest, RepositoryIngest):
batch.atoms.extend(repository_batch.atoms)
self.db.upsert_sources(batch.sources)
trust.stamp(batch.atoms, origin=origin)
self.db.upsert_atoms(batch.atoms)
self.vectors.sync_atoms([atom.atom_id for atom in batch.atoms])
+10 -2
View File
@@ -13,6 +13,8 @@ from datetime import datetime
from pathlib import Path
from typing import Any
from . import trust
from .cache_fabric import CacheFabric
from .config import Settings, get_settings
from .db import Database
from .schemas import (
@@ -45,6 +47,7 @@ class MemoryService:
self.db = database or Database(self.settings)
self.vectors = vectors or VectorIndex(self.settings, self.db)
self.pictures = VisualIndex(self.settings, self.db)
self.cache = CacheFabric(self.settings, self.db)
def remember(
self,
@@ -114,6 +117,7 @@ class MemoryService:
visibility=visibility or Visibility(),
retention_until=retention_until,
)
trust.stamp([atom], origin=origin)
self.db.upsert_atoms([atom])
self.vectors.sync_atoms([atom.atom_id])
return atom
@@ -169,8 +173,8 @@ class MemoryService:
) -> ForgetReport:
"""Delete a memory and everything derived from it.
The cascade is the whole point: an answer cached from a forgotten memory is still
on screen, and a vector index that still carries it will keep surfacing it.
The cascade is the whole point: a cached answer or a live vector will keep
surfacing a memory the owner asked to erase, so both go with it.
"""
if (atom_id is None) == (source_id is None):
raise ValueError("forget takes exactly one of atom_id or source_id")
@@ -185,6 +189,9 @@ class MemoryService:
edges_deleted = self.db.delete_edges_touching(doomed_ids)
atoms_deleted = self.db.delete_atoms(doomed_ids)
cache_cleared = self.db.clear_cache_events(f"forget:{target_source}")
semantic_cleared = self.cache.drop_semantic_cache()
if self.db.delete_source(target_source) != 1:
raise ValueError(f"{target_source}: the source row vanished before the forget could erase it")
vectors_dropped, visual_dropped = self._drop_derived_vectors(doomed_ids)
return ForgetReport(
source_id=target_source,
@@ -192,6 +199,7 @@ class MemoryService:
atoms_deleted=atoms_deleted,
edges_deleted=edges_deleted,
cache_entries_cleared=cache_cleared,
semantic_cache_cleared=semantic_cleared,
vectors_dropped=vectors_dropped,
visual_index_dropped=visual_dropped,
affected_cartridges=self._cartridges_holding(target_source),
+21 -2
View File
@@ -16,7 +16,7 @@ import sqlite3
from .storage import token_equivalent, visual_token_equivalent
from .text_index import index_text
SCHEMA_VERSION = 8
SCHEMA_VERSION = 9
ATOM_COLUMNS_V2 = """
atom_id TEXT PRIMARY KEY,
@@ -45,7 +45,8 @@ ATOM_COLUMNS_V2 = """
valid_to TEXT,
supersedes_json TEXT NOT NULL DEFAULT '[]',
visibility_json TEXT NOT NULL DEFAULT '{}',
retention_until TEXT
retention_until TEXT,
quarantined INTEGER NOT NULL DEFAULT 0
"""
_BASE_TABLES = """
@@ -133,6 +134,13 @@ _BASE_TABLES = """
value TEXT NOT NULL
);
-- One row, bumped by every write that changes which atoms are current. A compiled context is
-- only true for the corpus it was compiled from, and its cache key carries this number.
CREATE TABLE IF NOT EXISTS store_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE VIRTUAL TABLE IF NOT EXISTS atom_fts USING fts5(
atom_id UNINDEXED,
source,
@@ -174,6 +182,11 @@ _V1_TO_V2_COLUMNS = (
("retention_until", "TEXT"),
)
# Columns added after the second version, grouped by the schema version that introduced them.
_NEW_ATOM_COLUMNS: tuple[tuple[int, tuple[tuple[str, str], ...]], ...] = (
(9, (("quarantined", "INTEGER NOT NULL DEFAULT 0"),)),
)
def _table_exists(connection: sqlite3.Connection, name: str) -> bool:
row = connection.execute(
@@ -251,6 +264,12 @@ def apply(connection: sqlite3.Connection) -> None:
_rebuild_fts(connection)
if version < 7:
_recount_visual_tokens(connection)
for target, columns in _NEW_ATOM_COLUMNS:
if version < target:
existing = _column_names(connection, "atoms")
for name, definition in columns:
if name not in existing:
connection.execute(f"ALTER TABLE atoms ADD COLUMN {name} {definition}")
connection.executescript(VIEWS)
connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
+2
View File
@@ -69,6 +69,7 @@ def atom_params(atom: EvidenceAtom) -> tuple[Any, ...]:
to_json(atom.supersedes),
to_json(atom.visibility.model_dump()),
atom.retention_until.isoformat() if atom.retention_until else None,
1 if atom.quarantined else 0,
)
@@ -100,6 +101,7 @@ def row_to_atom(row: sqlite3.Row) -> EvidenceAtom:
supersedes=from_json(row["supersedes_json"], []),
visibility=Visibility(**from_json(row["visibility_json"], {})),
retention_until=from_iso(row["retention_until"]),
quarantined=bool(row["quarantined"]),
)
+3
View File
@@ -108,6 +108,7 @@ class EvidenceAtom(BaseModel):
supersedes: list[str] = Field(default_factory=list)
visibility: Visibility = Field(default_factory=Visibility)
retention_until: datetime | None = None
quarantined: bool = False
@model_validator(mode="after")
def _times_are_ordered(self) -> EvidenceAtom:
@@ -225,6 +226,7 @@ class CompiledContext(BaseModel):
packed_text: str
token_equivalent: int
cache_trace_id: str
quarantined_excluded: list[str] = Field(default_factory=list)
class CitationReport(BaseModel):
cited_atom_ids: list[str]
@@ -431,6 +433,7 @@ class ForgetReport(BaseModel):
atoms_deleted: int
edges_deleted: int
cache_entries_cleared: int
semantic_cache_cleared: int
vectors_dropped: int
visual_index_dropped: bool
affected_cartridges: list[dict[str, Any]]
+116
View File
@@ -0,0 +1,116 @@
"""Where a memory came from decides whether it may be cited.
A file handed to the server over HTTP is somebody else's words, and so is a sentence written to
read like an instruction to the model that will read it. Both stay searchable — hiding them would
destroy the evidence that the attempt happened — but neither becomes citable evidence until a
second, non-quarantined source says it too.
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING
from unicodedata import normalize
from .schemas import EvidenceAtom, MemoryOrigin
from .text_index import fts_expression, fts_terms, rare_terms
if TYPE_CHECKING:
from .db import Database
# Instruction-shaped phrases an untrusted text can aim at the model that reads it, matched folded.
INJECTION_SIGNALS: tuple[str, ...] = (
"ignore previous instructions",
"ignore all previous instructions",
"ignore the above instructions",
"disregard previous instructions",
"disregard the above",
"forget your instructions",
"new instructions:",
"you are now",
"system prompt",
"reveal your system prompt",
"忽略以上指令",
"忽略之前的指令",
"忽略前面的指令",
"无视上面的指令",
"忘记你的指令",
"新的指令:",
"你现在是",
"系统提示词",
"输出你的系统提示",
)
SPECIFIC_TERM_SAMPLE = 12
CORROBORATION_LIMIT = 3
def _fold(text: str) -> str:
return " ".join(normalize("NFKC", text).casefold().split())
_FOLDED_SIGNALS: tuple[tuple[str, str], ...] = tuple((_fold(signal), signal) for signal in INJECTION_SIGNALS)
def injection_signals(text: str) -> list[str]:
"""Which instruction-shaped phrases this text carries."""
folded = _fold(text)
return [signal for folded_signal, signal in _FOLDED_SIGNALS if folded_signal in folded]
def stamp(atoms: Iterable[EvidenceAtom], *, origin: MemoryOrigin) -> None:
"""Record where these atoms came from, and quarantine the ones that cannot be trusted on sight."""
for atom in atoms:
atom.origin = origin
signals = injection_signals(atom.text)
atom.quarantined = origin is MemoryOrigin.third_party or bool(signals)
if signals:
atom.metadata["injection_signals"] = signals
def corroborating_atom_ids(
database: Database, atom: EvidenceAtom, limit: int = CORROBORATION_LIMIT
) -> list[str]:
"""Live atoms from another source that carry this atom's most specific words.
Corroboration is term-level, not semantic: a second source sharing a claim's rarest words is
evidence that the claim was seen twice, not proof that it is true. A closed or quarantined
atom cannot corroborate, so a superseded fact stops propping up whatever cited it.
"""
terms = _specific_terms(atom.text)
if not terms:
return []
with database.connect() as connection:
rare = rare_terms(connection, terms)
if not rare:
return []
rows = connection.execute(
"""
SELECT atom_id
FROM atom_fts
WHERE atom_fts MATCH ?
AND atom_id IN (
SELECT atom_id FROM v_atoms_current
WHERE source_id != ? AND atom_id != ? AND quarantined = 0
)
ORDER BY bm25(atom_fts)
LIMIT ?
""",
(fts_expression(rare), atom.source_id, atom.atom_id, limit * 4),
).fetchall()
return [str(row["atom_id"]) for row in rows][:limit]
def is_citable(atom: EvidenceAtom, database: Database) -> bool:
"""A quarantined atom earns a citation only by matching a live, trusted atom from another source."""
return not atom.quarantined or bool(corroborating_atom_ids(database, atom))
def _specific_terms(text: str, limit: int = SPECIFIC_TERM_SAMPLE) -> list[str]:
"""The words most likely to identify this claim: the longest searchable terms it holds."""
try:
terms = fts_terms(text)
except ValueError:
return []
return sorted(dict.fromkeys(terms), key=lambda term: (-len(term), term))[:limit]