Require a token for writes and quarantine what nobody vouches for
Sixteen mutating routes now refuse a caller who cannot present the token the server started with; queries stay open, because a question is not a write. The environment variable is ELY_EYE_AUTH_TOKEN, following the repo's env_prefix convention — the shorter name I reached for first returned 403 where the smoke expected 401, and the server printed the reason. Words from a stranger are quarantined on the way in: an HTTP upload is third_party by definition, and a sentence carrying an instruction-shaped phrase is quarantined whoever wrote it. A quarantined atom stays findable and stays out of the compiled context until a second, live, unquarantined source agrees with its rarest words. Corroboration is computed at read time because it is a property of the corpus now, not of the moment a memory was written; it is term-level, and the docstring says so. forget now erases the source row and the compiled contexts beside the atoms, so a forgotten memory leaves nothing that can still speak. The semantic cache key carries a store revision, which fixes a real staleness: a context compiled before new evidence arrived kept its old quarantine decision forever. bench.audit --product still pointed at evals.py, which the p1-6 split moved into backend/src/ely_eye/proofs/. It failed fast, but the anti-self-justification rule had quietly stopped auditing the product; it now walks the package, and a test pins that the package exists and holds no re-roll. Gates: pytest 306 passed; ruff clean; mypy cold cache 24 errors in 9 files, the recorded baseline, net zero; pyright 0 errors on the 23 backend files and the 2 bench files; go build/vet/test and gofmt clean; npm run build 221.39 kB with eslint exit 0; bench/filesize_check clean; bench.audit --product clean over 31 harness files; scripts/mcp_write_smoke.sh PASS with the gate live across two servers and the MCP child.
This commit is contained in:
1 parent
8b974e6626
commit
9145a23f18
32 files changed
+750
-58
No files matched your search
@@ -13,6 +13,7 @@ from pydantic import BaseModel
|
||||
|
||||
from . import __version__
|
||||
from .adapters import plan as adapter_plan
|
||||
from .api_auth import require_write
|
||||
from .api_cartridges import register as register_cartridge_routes
|
||||
from .api_holders import register as register_holder_routes
|
||||
from .browse import browse, picture
|
||||
@@ -76,9 +77,10 @@ def settings_dep() -> Settings:
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(title="Ely-Eye", version=__version__)
|
||||
app.state.write_settings = settings_dep
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"],
|
||||
allow_origins=settings_dep().allowed_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
@@ -116,6 +118,7 @@ def create_app() -> FastAPI:
|
||||
def ingest_path(
|
||||
request: PathIngestRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
):
|
||||
try:
|
||||
return IngestionService(settings).ingest_path(request.path, request.cartridge_name)
|
||||
@@ -126,6 +129,7 @@ def create_app() -> FastAPI:
|
||||
def ingest_upload(
|
||||
metadata: Annotated[UploadIngestRequest, Depends()],
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
files: Annotated[list[UploadFile], File()],
|
||||
):
|
||||
upload_root = settings.data_dir / "uploads"
|
||||
@@ -144,7 +148,7 @@ def create_app() -> FastAPI:
|
||||
service = IngestionService(settings)
|
||||
aggregate = None
|
||||
for path in saved_paths:
|
||||
result = service.ingest_path(path, None)
|
||||
result = service.ingest_path(path, None, origin=MemoryOrigin.third_party)
|
||||
if aggregate is None:
|
||||
aggregate = result
|
||||
else:
|
||||
@@ -162,7 +166,11 @@ def create_app() -> FastAPI:
|
||||
return aggregate
|
||||
|
||||
@app.post("/api/chat", response_model=AnswerResponse)
|
||||
def chat(request: ChatRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> AnswerResponse:
|
||||
def chat(
|
||||
request: ChatRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> AnswerResponse:
|
||||
db = Database(settings)
|
||||
compiler = ContextCompiler(settings, db)
|
||||
context = compiler.compile(request.question, request.profile)
|
||||
@@ -200,7 +208,9 @@ def create_app() -> FastAPI:
|
||||
|
||||
@app.post("/api/memory", response_model=EvidenceAtom)
|
||||
def remember(
|
||||
request: RememberRequest, settings: Annotated[Settings, Depends(settings_dep)]
|
||||
request: RememberRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> EvidenceAtom:
|
||||
try:
|
||||
return MemoryService(settings).remember(
|
||||
@@ -218,7 +228,9 @@ def create_app() -> FastAPI:
|
||||
|
||||
@app.post("/api/memory/supersede", response_model=EvidenceAtom)
|
||||
def supersede(
|
||||
request: SupersedeRequest, settings: Annotated[Settings, Depends(settings_dep)]
|
||||
request: SupersedeRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> EvidenceAtom:
|
||||
try:
|
||||
return MemoryService(settings).supersede(
|
||||
@@ -234,7 +246,11 @@ def create_app() -> FastAPI:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/memory/forget", response_model=ForgetReport)
|
||||
def forget(request: ForgetRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> ForgetReport:
|
||||
def forget(
|
||||
request: ForgetRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> ForgetReport:
|
||||
try:
|
||||
return MemoryService(settings).forget(
|
||||
atom_id=request.atom_id, source_id=request.source_id, reason=request.reason
|
||||
@@ -288,7 +304,11 @@ def create_app() -> FastAPI:
|
||||
return adapter_plan(settings, Database(settings))
|
||||
|
||||
@app.post("/api/hashhop", response_model=HashHopProof)
|
||||
def hashhop(request: HashHopRequest, settings: Annotated[Settings, Depends(settings_dep)]) -> HashHopProof:
|
||||
def hashhop(
|
||||
request: HashHopRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> HashHopProof:
|
||||
evaluator = HashHopEvaluator(settings)
|
||||
if request.kind == "text":
|
||||
return evaluator.generate_text_proof(request.hops, request.token_equivalent)
|
||||
@@ -300,6 +320,7 @@ def create_app() -> FastAPI:
|
||||
def proof_suite(
|
||||
request: ProofSuiteRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> ProofSuiteReport:
|
||||
try:
|
||||
return PRDProofSuite(settings).run(request.cartridge_id)
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Writes need a token; queries do not.
|
||||
|
||||
Anything that can change stored state — a memory, an ingested source, a mounted cartridge, a
|
||||
proof run — is refused unless the caller presents the token the server was started with. A
|
||||
browser must also come from an allowed origin. The CLI and the MCP server reach the same
|
||||
writer through the same rule, so no face of the product gets a cheaper path to the store.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .config import Settings
|
||||
|
||||
|
||||
def require_write(request: Request) -> None:
|
||||
"""Gate a mutating request. The app carries the settings source, so this stays a plain global."""
|
||||
|
||||
settings: Settings = request.app.state.write_settings()
|
||||
origin = request.headers.get("origin")
|
||||
if origin is not None and origin not in settings.allowed_origins:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"origin {origin} may not write; allowed origins: {', '.join(settings.allowed_origins)}",
|
||||
)
|
||||
configured = settings.auth_token
|
||||
if not configured:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="this server accepts no writes: start it with ELY_EYE_AUTH_TOKEN set",
|
||||
)
|
||||
header = request.headers.get("authorization", "")
|
||||
presented = header.removeprefix("Bearer ").strip() if header.startswith("Bearer ") else ""
|
||||
if not presented or not secrets.compare_digest(presented, configured):
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="write needs the token: send Authorization: Bearer $ELY_EYE_AUTH_TOKEN",
|
||||
)
|
||||
@@ -7,6 +7,7 @@ from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException
|
||||
|
||||
from .api_auth import require_write
|
||||
from .cartridge import CartridgeService
|
||||
from .cartridge_assets import CartridgeAssetService
|
||||
from .config import Settings
|
||||
@@ -17,6 +18,9 @@ from .schemas import AppendRequest, CartridgeAssetReport, CartridgeManifest, Mou
|
||||
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
"""Mounting decides what a search may see; the KV fingerprint decides if it may mount here."""
|
||||
|
||||
app.state.write_settings = settings_dep
|
||||
|
||||
|
||||
@app.get("/api/cartridges")
|
||||
def cartridges(settings: Annotated[Settings, Depends(settings_dep)]):
|
||||
return CartridgeService(settings).list_manifests()
|
||||
@@ -29,6 +33,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
def mount_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> CartridgeManifest:
|
||||
try:
|
||||
return CartridgeService(settings).mount(cartridge_id)
|
||||
@@ -39,6 +44,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
def switch_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> CartridgeManifest:
|
||||
try:
|
||||
return CartridgeService(settings).switch(cartridge_id)
|
||||
@@ -49,6 +55,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
def unmount_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> MountState:
|
||||
try:
|
||||
return CartridgeService(settings).unmount(cartridge_id)
|
||||
@@ -60,6 +67,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
cartridge_id: str,
|
||||
request: AppendRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> CartridgeManifest:
|
||||
try:
|
||||
return CartridgeService(settings).append(cartridge_id, request.atom_ids)
|
||||
@@ -70,6 +78,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
def finalize_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> CartridgeAssetReport:
|
||||
try:
|
||||
return CartridgeAssetService(settings).finalize(cartridge_id)
|
||||
@@ -80,6 +89,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
def refresh_cartridge(
|
||||
cartridge_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
):
|
||||
db = Database(settings)
|
||||
try:
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException
|
||||
|
||||
from .api_auth import require_write
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .holder_store import HolderStore
|
||||
@@ -28,9 +29,12 @@ from .transactive_schemas import (
|
||||
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
"""Register the transactive routes. The server owns the timestamps, the caller does not."""
|
||||
|
||||
app.state.write_settings = settings_dep
|
||||
|
||||
def store(settings: Settings) -> HolderStore:
|
||||
return HolderStore(Database(settings))
|
||||
|
||||
|
||||
@app.get("/api/holders", response_model=list[Holder])
|
||||
def holders(settings: Annotated[Settings, Depends(settings_dep)]) -> list[Holder]:
|
||||
return store(settings).list_holders()
|
||||
@@ -40,6 +44,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
holder_id: str,
|
||||
request: HolderUpsertRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> Holder:
|
||||
now = utc_now()
|
||||
entry = Holder(
|
||||
@@ -67,6 +72,7 @@ def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
def delete_holder(
|
||||
holder_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
_: Annotated[None, Depends(require_write)],
|
||||
) -> Holder:
|
||||
registry = store(settings)
|
||||
entry = registry.get(holder_id)
|
||||
|
||||
@@ -17,8 +17,9 @@ class CacheFabric:
|
||||
self.settings = settings or get_settings()
|
||||
self.db = database or Database(self.settings)
|
||||
|
||||
def semantic_key(self, question: str, profile: str) -> str:
|
||||
return hashlib.sha256(f"compiler-v2\n{profile}\n{question}".encode()).hexdigest()
|
||||
def semantic_key(self, question: str, profile: str, store_revision: int) -> str:
|
||||
"""A compiled context is only true for the corpus it was compiled from, so the corpus is part of its name."""
|
||||
return hashlib.sha256(f"compiler-v2\n{profile}\n{store_revision}\n{question}".encode()).hexdigest()
|
||||
|
||||
def record_hit(self, layer: str, cache_key: str, bytes_estimate: int = 0, metadata: dict[str, Any] | None = None) -> None:
|
||||
self.db.log_cache_event(layer, cache_key, "hit", bytes_estimate, metadata)
|
||||
@@ -43,6 +44,16 @@ class CacheFabric:
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise ValueError(f"{path}: the semantic cache entry is unreadable ({error})") from error
|
||||
|
||||
def drop_semantic_cache(self) -> int:
|
||||
"""Delete every compiled context: each one may quote a memory that no longer exists."""
|
||||
directory = self.settings.cache_dir / "semantic"
|
||||
if not directory.exists():
|
||||
return 0
|
||||
entries = list(directory.glob("*.json"))
|
||||
for path in entries:
|
||||
path.unlink()
|
||||
return len(entries)
|
||||
|
||||
def statuses(self) -> list[CacheLayerStatus]:
|
||||
rows = self.db.cache_layer_rows()
|
||||
statuses: list[CacheLayerStatus] = []
|
||||
|
||||
@@ -46,6 +46,11 @@ register_maintenance_commands(app, lambda: get_settings())
|
||||
|
||||
@app.command()
|
||||
def serve(host: str = "127.0.0.1", port: int = 8765) -> None:
|
||||
settings = get_settings()
|
||||
if settings.auth_token:
|
||||
typer.echo(f"writes need Authorization: Bearer $ELY_EYE_AUTH_TOKEN; {len(settings.allowed_origins)} allowed origin(s)")
|
||||
else:
|
||||
typer.echo("writes are closed on this server: start it with ELY_EYE_AUTH_TOKEN set")
|
||||
uvicorn.run(create_app(), host=host, port=port)
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
from collections.abc import Iterable
|
||||
|
||||
from . import trust
|
||||
from .cache_fabric import SEMANTIC_LAYER, CacheFabric
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
@@ -25,7 +26,7 @@ class ContextCompiler:
|
||||
self.cache = CacheFabric(self.settings, self.db)
|
||||
|
||||
def compile(self, question: str, profile: RuntimeProfile) -> CompiledContext:
|
||||
cache_key = self.cache.semantic_key(question, profile.value)
|
||||
cache_key = self.cache.semantic_key(question, profile.value, self.db.revision())
|
||||
cached = self.cache.read_semantic_cache(cache_key)
|
||||
if cached is not None:
|
||||
compiled = CompiledContext.model_validate(cached)
|
||||
@@ -40,7 +41,8 @@ class ContextCompiler:
|
||||
self.cache.record_miss(SEMANTIC_LAYER, cache_key)
|
||||
plan = self.plan(question, profile)
|
||||
hits = self.retrieval.search(plan.retrieval_query, plan.evidence_budget_atoms).hits
|
||||
packed_text, token_total = self._pack_hits(hits, plan.token_budget, plan.retrieval_query)
|
||||
citable, held = self._split_citable(hits)
|
||||
packed_text, token_total = self._pack_hits(citable, plan.token_budget, plan.retrieval_query)
|
||||
trace_id = hashlib.sha256(f"{cache_key}:{token_total}".encode()).hexdigest()[:16]
|
||||
compiled = CompiledContext(
|
||||
plan=plan,
|
||||
@@ -48,6 +50,7 @@ class ContextCompiler:
|
||||
packed_text=packed_text,
|
||||
token_equivalent=token_total,
|
||||
cache_trace_id=trace_id,
|
||||
quarantined_excluded=[hit.atom.atom_id for hit in held],
|
||||
)
|
||||
self.cache.write_semantic_cache(cache_key, compiled.model_dump(mode="json"))
|
||||
return compiled
|
||||
@@ -71,6 +74,15 @@ class ContextCompiler:
|
||||
],
|
||||
)
|
||||
|
||||
def _split_citable(self, hits: list[RetrievalHit]) -> tuple[list[RetrievalHit], list[RetrievalHit]]:
|
||||
"""A quarantined atom reaches the prompt only when a trusted source says the same thing."""
|
||||
|
||||
citable: list[RetrievalHit] = []
|
||||
held: list[RetrievalHit] = []
|
||||
for hit in hits:
|
||||
(citable if trust.is_citable(hit.atom, self.db) else held).append(hit)
|
||||
return citable, held
|
||||
|
||||
def _compression_strategy(self, profile: RuntimeProfile, modalities: Iterable[Modality]) -> str:
|
||||
has_visual = any(modality in {Modality.image, Modality.pdf_page, Modality.video_frame, Modality.ui_screenshot} for modality in modalities)
|
||||
if profile == RuntimeProfile.extreme_context:
|
||||
|
||||
@@ -13,6 +13,10 @@ class Settings(BaseSettings):
|
||||
|
||||
home: Path = Field(default_factory=lambda: Path.cwd() / ".ely_eye")
|
||||
default_holder: str = "local"
|
||||
auth_token: str | None = None
|
||||
allowed_origins: list[str] = Field(
|
||||
default_factory=lambda: ["http://localhost:5173", "http://127.0.0.1:5173"]
|
||||
)
|
||||
model_id: str = "Qwen/Qwen3.5-9B"
|
||||
model_dir: Path | None = None
|
||||
kv_cache_dtype: str = "auto"
|
||||
|
||||
@@ -81,9 +81,10 @@ class Database:
|
||||
embedding_refs_json, layout_json, relations_json, trust_json,
|
||||
token_equivalent, metadata_json, holder_id, origin, scenario,
|
||||
event_time, event_time_end, event_time_precision, event_time_source,
|
||||
valid_from, valid_to, supersedes_json, visibility_json, retention_until
|
||||
valid_from, valid_to, supersedes_json, visibility_json, retention_until,
|
||||
quarantined
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(atom_id) DO UPDATE SET
|
||||
modality=excluded.modality,
|
||||
source=excluded.source,
|
||||
@@ -107,7 +108,8 @@ class Database:
|
||||
valid_to=excluded.valid_to,
|
||||
supersedes_json=excluded.supersedes_json,
|
||||
visibility_json=excluded.visibility_json,
|
||||
retention_until=excluded.retention_until
|
||||
retention_until=excluded.retention_until,
|
||||
quarantined=excluded.quarantined
|
||||
""",
|
||||
[atom_params(atom) for atom in atoms],
|
||||
)
|
||||
@@ -118,6 +120,7 @@ class Database:
|
||||
)
|
||||
# New text changes which words are common, so every remembered count is void.
|
||||
connection.execute("DELETE FROM term_df")
|
||||
self._bump_revision(connection)
|
||||
|
||||
def list_atoms(self, limit: int | None = None) -> list[EvidenceAtom]:
|
||||
with self.connect() as connection:
|
||||
@@ -188,6 +191,7 @@ class Database:
|
||||
"WHERE valid_to IS NULL AND atom_id IN (SELECT value FROM json_each(?))",
|
||||
(closed_at.isoformat(), to_json(atom_ids)),
|
||||
)
|
||||
self._bump_revision(connection)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
def delete_atoms(self, atom_ids: list[str]) -> int:
|
||||
@@ -201,6 +205,13 @@ class Database:
|
||||
"DELETE FROM atoms WHERE atom_id IN (SELECT value FROM json_each(?))", (to_json(atom_ids),)
|
||||
)
|
||||
connection.execute("DELETE FROM term_df")
|
||||
self._bump_revision(connection)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
def delete_source(self, source_id: str) -> int:
|
||||
"""Erase where a memory came from, so a forget leaves no provenance row behind."""
|
||||
with self.connect() as connection:
|
||||
cursor = connection.execute("DELETE FROM sources WHERE source_id = ?", (source_id,))
|
||||
return int(cursor.rowcount)
|
||||
|
||||
def delete_edges_touching(self, atom_ids: list[str]) -> int:
|
||||
@@ -231,6 +242,17 @@ class Database:
|
||||
with self.connect() as connection:
|
||||
return int(connection.execute("SELECT COUNT(*) FROM atoms").fetchone()[0])
|
||||
|
||||
def revision(self) -> int:
|
||||
"""What a compiled context was compiled against; see store_meta."""
|
||||
with self.connect() as connection:
|
||||
row = connection.execute("SELECT value FROM store_meta WHERE key = 'revision'").fetchone()
|
||||
return int(row["value"]) if row is not None else 0
|
||||
|
||||
def _bump_revision(self, connection: sqlite3.Connection) -> None:
|
||||
"""A write changes what a compiled context may contain, so the semantic cache must notice."""
|
||||
connection.execute("INSERT OR IGNORE INTO store_meta(key, value) VALUES ('revision', '0')")
|
||||
connection.execute("UPDATE store_meta SET value = CAST(value AS INTEGER) + 1 WHERE key = 'revision'")
|
||||
|
||||
def token_equivalent_sum(self) -> int:
|
||||
with self.connect() as connection:
|
||||
value = connection.execute("SELECT COALESCE(SUM(token_equivalent), 0) FROM atoms").fetchone()[0]
|
||||
|
||||
@@ -6,6 +6,7 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import trust
|
||||
from .asr import SpeechTranscriber
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
@@ -25,6 +26,7 @@ from .ingestion_repository import RepositoryIngest, extract_symbols
|
||||
from .schemas import (
|
||||
EvidenceAtom,
|
||||
IngestResponse,
|
||||
MemoryOrigin,
|
||||
Modality,
|
||||
SkippedFile,
|
||||
SourceRecord,
|
||||
@@ -55,7 +57,13 @@ class IngestionService(MediaIngest, RepositoryIngest):
|
||||
self.transcriber = SpeechTranscriber(self.settings)
|
||||
self.vectors = vectors or VectorIndex(self.settings, self.db)
|
||||
|
||||
def ingest_path(self, path: Path, cartridge_name: str | None = None) -> IngestResponse:
|
||||
def ingest_path(
|
||||
self,
|
||||
path: Path,
|
||||
cartridge_name: str | None = None,
|
||||
*,
|
||||
origin: MemoryOrigin = MemoryOrigin.artifact,
|
||||
) -> IngestResponse:
|
||||
path = path.expanduser().resolve()
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(path)
|
||||
@@ -79,6 +87,7 @@ class IngestionService(MediaIngest, RepositoryIngest):
|
||||
batch.atoms.extend(repository_batch.atoms)
|
||||
|
||||
self.db.upsert_sources(batch.sources)
|
||||
trust.stamp(batch.atoms, origin=origin)
|
||||
self.db.upsert_atoms(batch.atoms)
|
||||
self.vectors.sync_atoms([atom.atom_id for atom in batch.atoms])
|
||||
|
||||
|
||||
@@ -13,6 +13,8 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import trust
|
||||
from .cache_fabric import CacheFabric
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .schemas import (
|
||||
@@ -45,6 +47,7 @@ class MemoryService:
|
||||
self.db = database or Database(self.settings)
|
||||
self.vectors = vectors or VectorIndex(self.settings, self.db)
|
||||
self.pictures = VisualIndex(self.settings, self.db)
|
||||
self.cache = CacheFabric(self.settings, self.db)
|
||||
|
||||
def remember(
|
||||
self,
|
||||
@@ -114,6 +117,7 @@ class MemoryService:
|
||||
visibility=visibility or Visibility(),
|
||||
retention_until=retention_until,
|
||||
)
|
||||
trust.stamp([atom], origin=origin)
|
||||
self.db.upsert_atoms([atom])
|
||||
self.vectors.sync_atoms([atom.atom_id])
|
||||
return atom
|
||||
@@ -169,8 +173,8 @@ class MemoryService:
|
||||
) -> ForgetReport:
|
||||
"""Delete a memory and everything derived from it.
|
||||
|
||||
The cascade is the whole point: an answer cached from a forgotten memory is still
|
||||
on screen, and a vector index that still carries it will keep surfacing it.
|
||||
The cascade is the whole point: a cached answer or a live vector will keep
|
||||
surfacing a memory the owner asked to erase, so both go with it.
|
||||
"""
|
||||
if (atom_id is None) == (source_id is None):
|
||||
raise ValueError("forget takes exactly one of atom_id or source_id")
|
||||
@@ -185,6 +189,9 @@ class MemoryService:
|
||||
edges_deleted = self.db.delete_edges_touching(doomed_ids)
|
||||
atoms_deleted = self.db.delete_atoms(doomed_ids)
|
||||
cache_cleared = self.db.clear_cache_events(f"forget:{target_source}")
|
||||
semantic_cleared = self.cache.drop_semantic_cache()
|
||||
if self.db.delete_source(target_source) != 1:
|
||||
raise ValueError(f"{target_source}: the source row vanished before the forget could erase it")
|
||||
vectors_dropped, visual_dropped = self._drop_derived_vectors(doomed_ids)
|
||||
return ForgetReport(
|
||||
source_id=target_source,
|
||||
@@ -192,6 +199,7 @@ class MemoryService:
|
||||
atoms_deleted=atoms_deleted,
|
||||
edges_deleted=edges_deleted,
|
||||
cache_entries_cleared=cache_cleared,
|
||||
semantic_cache_cleared=semantic_cleared,
|
||||
vectors_dropped=vectors_dropped,
|
||||
visual_index_dropped=visual_dropped,
|
||||
affected_cartridges=self._cartridges_holding(target_source),
|
||||
|
||||
@@ -16,7 +16,7 @@ import sqlite3
|
||||
from .storage import token_equivalent, visual_token_equivalent
|
||||
from .text_index import index_text
|
||||
|
||||
SCHEMA_VERSION = 8
|
||||
SCHEMA_VERSION = 9
|
||||
|
||||
ATOM_COLUMNS_V2 = """
|
||||
atom_id TEXT PRIMARY KEY,
|
||||
@@ -45,7 +45,8 @@ ATOM_COLUMNS_V2 = """
|
||||
valid_to TEXT,
|
||||
supersedes_json TEXT NOT NULL DEFAULT '[]',
|
||||
visibility_json TEXT NOT NULL DEFAULT '{}',
|
||||
retention_until TEXT
|
||||
retention_until TEXT,
|
||||
quarantined INTEGER NOT NULL DEFAULT 0
|
||||
"""
|
||||
|
||||
_BASE_TABLES = """
|
||||
@@ -133,6 +134,13 @@ _BASE_TABLES = """
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- One row, bumped by every write that changes which atoms are current. A compiled context is
|
||||
-- only true for the corpus it was compiled from, and its cache key carries this number.
|
||||
CREATE TABLE IF NOT EXISTS store_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS atom_fts USING fts5(
|
||||
atom_id UNINDEXED,
|
||||
source,
|
||||
@@ -174,6 +182,11 @@ _V1_TO_V2_COLUMNS = (
|
||||
("retention_until", "TEXT"),
|
||||
)
|
||||
|
||||
# Columns added after the second version, grouped by the schema version that introduced them.
|
||||
_NEW_ATOM_COLUMNS: tuple[tuple[int, tuple[tuple[str, str], ...]], ...] = (
|
||||
(9, (("quarantined", "INTEGER NOT NULL DEFAULT 0"),)),
|
||||
)
|
||||
|
||||
|
||||
def _table_exists(connection: sqlite3.Connection, name: str) -> bool:
|
||||
row = connection.execute(
|
||||
@@ -251,6 +264,12 @@ def apply(connection: sqlite3.Connection) -> None:
|
||||
_rebuild_fts(connection)
|
||||
if version < 7:
|
||||
_recount_visual_tokens(connection)
|
||||
for target, columns in _NEW_ATOM_COLUMNS:
|
||||
if version < target:
|
||||
existing = _column_names(connection, "atoms")
|
||||
for name, definition in columns:
|
||||
if name not in existing:
|
||||
connection.execute(f"ALTER TABLE atoms ADD COLUMN {name} {definition}")
|
||||
|
||||
connection.executescript(VIEWS)
|
||||
connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
||||
|
||||
@@ -69,6 +69,7 @@ def atom_params(atom: EvidenceAtom) -> tuple[Any, ...]:
|
||||
to_json(atom.supersedes),
|
||||
to_json(atom.visibility.model_dump()),
|
||||
atom.retention_until.isoformat() if atom.retention_until else None,
|
||||
1 if atom.quarantined else 0,
|
||||
)
|
||||
|
||||
|
||||
@@ -100,6 +101,7 @@ def row_to_atom(row: sqlite3.Row) -> EvidenceAtom:
|
||||
supersedes=from_json(row["supersedes_json"], []),
|
||||
visibility=Visibility(**from_json(row["visibility_json"], {})),
|
||||
retention_until=from_iso(row["retention_until"]),
|
||||
quarantined=bool(row["quarantined"]),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -108,6 +108,7 @@ class EvidenceAtom(BaseModel):
|
||||
supersedes: list[str] = Field(default_factory=list)
|
||||
visibility: Visibility = Field(default_factory=Visibility)
|
||||
retention_until: datetime | None = None
|
||||
quarantined: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _times_are_ordered(self) -> EvidenceAtom:
|
||||
@@ -225,6 +226,7 @@ class CompiledContext(BaseModel):
|
||||
packed_text: str
|
||||
token_equivalent: int
|
||||
cache_trace_id: str
|
||||
quarantined_excluded: list[str] = Field(default_factory=list)
|
||||
|
||||
class CitationReport(BaseModel):
|
||||
cited_atom_ids: list[str]
|
||||
@@ -431,6 +433,7 @@ class ForgetReport(BaseModel):
|
||||
atoms_deleted: int
|
||||
edges_deleted: int
|
||||
cache_entries_cleared: int
|
||||
semantic_cache_cleared: int
|
||||
vectors_dropped: int
|
||||
visual_index_dropped: bool
|
||||
affected_cartridges: list[dict[str, Any]]
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Where a memory came from decides whether it may be cited.
|
||||
|
||||
A file handed to the server over HTTP is somebody else's words, and so is a sentence written to
|
||||
read like an instruction to the model that will read it. Both stay searchable — hiding them would
|
||||
destroy the evidence that the attempt happened — but neither becomes citable evidence until a
|
||||
second, non-quarantined source says it too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
from unicodedata import normalize
|
||||
|
||||
from .schemas import EvidenceAtom, MemoryOrigin
|
||||
from .text_index import fts_expression, fts_terms, rare_terms
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .db import Database
|
||||
|
||||
# Instruction-shaped phrases an untrusted text can aim at the model that reads it, matched folded.
|
||||
INJECTION_SIGNALS: tuple[str, ...] = (
|
||||
"ignore previous instructions",
|
||||
"ignore all previous instructions",
|
||||
"ignore the above instructions",
|
||||
"disregard previous instructions",
|
||||
"disregard the above",
|
||||
"forget your instructions",
|
||||
"new instructions:",
|
||||
"you are now",
|
||||
"system prompt",
|
||||
"reveal your system prompt",
|
||||
"忽略以上指令",
|
||||
"忽略之前的指令",
|
||||
"忽略前面的指令",
|
||||
"无视上面的指令",
|
||||
"忘记你的指令",
|
||||
"新的指令:",
|
||||
"你现在是",
|
||||
"系统提示词",
|
||||
"输出你的系统提示",
|
||||
)
|
||||
|
||||
SPECIFIC_TERM_SAMPLE = 12
|
||||
CORROBORATION_LIMIT = 3
|
||||
|
||||
|
||||
def _fold(text: str) -> str:
|
||||
return " ".join(normalize("NFKC", text).casefold().split())
|
||||
|
||||
|
||||
_FOLDED_SIGNALS: tuple[tuple[str, str], ...] = tuple((_fold(signal), signal) for signal in INJECTION_SIGNALS)
|
||||
|
||||
|
||||
def injection_signals(text: str) -> list[str]:
|
||||
"""Which instruction-shaped phrases this text carries."""
|
||||
folded = _fold(text)
|
||||
return [signal for folded_signal, signal in _FOLDED_SIGNALS if folded_signal in folded]
|
||||
|
||||
|
||||
def stamp(atoms: Iterable[EvidenceAtom], *, origin: MemoryOrigin) -> None:
|
||||
"""Record where these atoms came from, and quarantine the ones that cannot be trusted on sight."""
|
||||
|
||||
for atom in atoms:
|
||||
atom.origin = origin
|
||||
signals = injection_signals(atom.text)
|
||||
atom.quarantined = origin is MemoryOrigin.third_party or bool(signals)
|
||||
if signals:
|
||||
atom.metadata["injection_signals"] = signals
|
||||
|
||||
|
||||
def corroborating_atom_ids(
|
||||
database: Database, atom: EvidenceAtom, limit: int = CORROBORATION_LIMIT
|
||||
) -> list[str]:
|
||||
"""Live atoms from another source that carry this atom's most specific words.
|
||||
|
||||
Corroboration is term-level, not semantic: a second source sharing a claim's rarest words is
|
||||
evidence that the claim was seen twice, not proof that it is true. A closed or quarantined
|
||||
atom cannot corroborate, so a superseded fact stops propping up whatever cited it.
|
||||
"""
|
||||
terms = _specific_terms(atom.text)
|
||||
if not terms:
|
||||
return []
|
||||
with database.connect() as connection:
|
||||
rare = rare_terms(connection, terms)
|
||||
if not rare:
|
||||
return []
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT atom_id
|
||||
FROM atom_fts
|
||||
WHERE atom_fts MATCH ?
|
||||
AND atom_id IN (
|
||||
SELECT atom_id FROM v_atoms_current
|
||||
WHERE source_id != ? AND atom_id != ? AND quarantined = 0
|
||||
)
|
||||
ORDER BY bm25(atom_fts)
|
||||
LIMIT ?
|
||||
""",
|
||||
(fts_expression(rare), atom.source_id, atom.atom_id, limit * 4),
|
||||
).fetchall()
|
||||
return [str(row["atom_id"]) for row in rows][:limit]
|
||||
|
||||
|
||||
def is_citable(atom: EvidenceAtom, database: Database) -> bool:
|
||||
"""A quarantined atom earns a citation only by matching a live, trusted atom from another source."""
|
||||
return not atom.quarantined or bool(corroborating_atom_ids(database, atom))
|
||||
|
||||
|
||||
def _specific_terms(text: str, limit: int = SPECIFIC_TERM_SAMPLE) -> list[str]:
|
||||
"""The words most likely to identify this claim: the longest searchable terms it holds."""
|
||||
try:
|
||||
terms = fts_terms(text)
|
||||
except ValueError:
|
||||
return []
|
||||
return sorted(dict.fromkeys(terms), key=lambda term: (-len(term), term))[:limit]
|
||||
Reference in new issue
Block a user