Give memories a working tier, a long-term tier, and a record of what disagrees
A memory used to have one state: present. A session could not be reloaded
without searching, nothing separated what a person said today from what the
store ingested a year ago, and two statements that disagree about a number
were both handed to the reader as if the store believed both.
Schema v10. Atoms gain session_id, importance, recall_count, last_recalled_at
and archived_at; two tables arrive beside them — consolidations (one watermark
per holder+scenario lane) and conflicts (a pair, the term they disagree on,
both values, and whether anyone has settled it). v_atoms_current now also
hides archived rows, and apply() drops the view before rebuilding it because
CREATE VIEW IF NOT EXISTS would keep the old body and ignore the archive.
Working memory is a read, not a store: `working <holder> <session>` returns
what that session recorded, in order, with no search involved. Consolidation
rolls a lane's new memories into one standing summary and moves the watermark
to the newest memory actually rolled up — not to the clock of the pass, which
made a second pass count the same memories twice. The archive rule is stated
in the command that runs it (recorded before N days, never returned by a
search, importance at or under a floor) rather than hidden in a decay formula
nobody measured; --dry-run names the victims first.
A conflict is recorded when a new memory repeats a label with a different
number in the same holder's same scenario. Retrieval drops the older side
while the conflict is open and both sides answer again once it is settled:
settling is not superseding, and a word-only disagreement is left alone
because a reworded sentence would fire on every paraphrase. Numbers are keyed
to both the memory's subject and the nearest preceding label, measured: the
two thermostat sentences share only their subject, so a nearest-word-only rule
found nothing.
Search now counts the memories it handed back, which is the only use estimate
the store has, and the archive rule reads it. The dense index applies the same
currency rules as the view, so an archived memory cannot stay findable through
one leg while the other hides it.
Two reads over HTTP (working, conflicts) and two tools over MCP
(working_set, memory_conflicts); curation stays with the operator because an
agent should not decide what its own past is worth. remember and supersede
carry a session id; importance does not cross MCP, because recall_count is
observed and importance is stated.
Gates: 366 passed; ruff clean; mypy 24 errors in 9 files, the recorded 9cb46c1
baseline, net 0; pyright 0 on the touched files; go build/vet/test and gofmt
clean; mcp_write_smoke PASS.
This commit is contained in:
1 parent
4c6af87fe0
commit
dc9cfb5a30
30 files changed
+1814
-229
No files matched your search
@@ -16,6 +16,7 @@ from .adapters import plan as adapter_plan
|
||||
from .api_auth import require_write
|
||||
from .api_cartridges import register as register_cartridge_routes
|
||||
from .api_holders import register as register_holder_routes
|
||||
from .api_memory import register as register_memory_routes
|
||||
from .browse import browse, picture
|
||||
from .cache_fabric import CacheFabric
|
||||
from .cartridge import CartridgeService
|
||||
@@ -219,6 +220,8 @@ def create_app() -> FastAPI:
|
||||
event_time_precision=request.event_time_precision,
|
||||
event_time_source=request.event_time_source,
|
||||
retention_until=request.retention_until,
|
||||
session_id=request.session_id,
|
||||
importance=request.importance,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
@@ -238,6 +241,8 @@ def create_app() -> FastAPI:
|
||||
event_time=request.event_time,
|
||||
event_time_precision=request.event_time_precision,
|
||||
event_time_source=request.event_time_source,
|
||||
session_id=request.session_id,
|
||||
importance=request.importance,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
@@ -257,6 +262,7 @@ def create_app() -> FastAPI:
|
||||
|
||||
register_cartridge_routes(app, settings_dep)
|
||||
register_holder_routes(app, settings_dep)
|
||||
register_memory_routes(app, settings_dep)
|
||||
|
||||
@app.get("/api/adapters", response_model=list[AdapterStatus])
|
||||
def adapters(settings: Annotated[Settings, Depends(settings_dep)]) -> list[AdapterStatus]:
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""The memory tiers over HTTP: what a session holds, and what the store disagrees about.
|
||||
|
||||
Only the two reads an agent needs are exposed. Curation — rolling a lane up, archiving what
|
||||
nobody uses, settling a disagreement — is the holder's decision, so it lives in the CLI and the
|
||||
service, not in a route with no caller.
|
||||
|
||||
No `from __future__ import annotations` here: FastAPI reads these route signatures, and string
|
||||
annotations would be resolved against module globals, where the injected settings_dep exists
|
||||
only as a parameter of register().
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException
|
||||
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .memory import MemoryService
|
||||
from .schemas import ConflictRecord, EvidenceAtom
|
||||
|
||||
|
||||
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
"""Register the tier reads. The holder is named by the caller because the question is about
|
||||
somebody's memory, not about the server's own identity."""
|
||||
|
||||
def service(settings: Settings) -> MemoryService:
|
||||
return MemoryService(settings, Database(settings))
|
||||
|
||||
@app.get("/api/memory/working", response_model=list[EvidenceAtom])
|
||||
def working_set(
|
||||
holder: str,
|
||||
session: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> list[EvidenceAtom]:
|
||||
"""What one session recorded, in the order it recorded it, with no search involved."""
|
||||
try:
|
||||
return service(settings).working_set(holder, session)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
@app.get("/api/memory/conflicts", response_model=list[ConflictRecord])
|
||||
def conflicts(
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
holder: str | None = None,
|
||||
) -> list[ConflictRecord]:
|
||||
"""Every disagreement the store noticed and nobody has settled."""
|
||||
return service(settings).conflicts(holder)
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
@@ -17,15 +16,15 @@ from .benchmarks import (
|
||||
)
|
||||
from .cartridge import CartridgeService
|
||||
from .cartridge_assets import CartridgeAssetService
|
||||
from .cli_common import json_dump
|
||||
from .cli_common import json_dump, moment
|
||||
from .cli_maintenance import register as register_maintenance_commands
|
||||
from .cli_memory import register as register_memory_commands
|
||||
from .cli_senses import register as register_sense_commands
|
||||
from .cli_senses import senses_report
|
||||
from .cli_transactive import register as register_transactive_commands
|
||||
from .config import get_settings
|
||||
from .db import Database
|
||||
from .ingestion import IngestionService
|
||||
from .memory import MemoryService
|
||||
from .profiles import runtime_profile_report
|
||||
from .proofs import HashHopEvaluator, PRDProofSuite
|
||||
from .proofs.contradictions import CORPUS_NAME, build_contradiction_scenarios, write_corpus
|
||||
@@ -33,7 +32,7 @@ from .proofs.table import render_table
|
||||
from .research_modules import ResearchModuleService
|
||||
from .retrieval import RetrievalService
|
||||
from .runtime import QwenRuntime
|
||||
from .schemas import MemoryOrigin, SearchFilters, TimePrecision, TrainingAdapterKind
|
||||
from .schemas import SearchFilters, TrainingAdapterKind
|
||||
from .training import TrainingService
|
||||
|
||||
app = typer.Typer(no_args_is_help=True)
|
||||
@@ -42,6 +41,7 @@ app = typer.Typer(no_args_is_help=True)
|
||||
register_transactive_commands(app, lambda: get_settings())
|
||||
register_sense_commands(app, lambda: get_settings())
|
||||
register_maintenance_commands(app, lambda: get_settings())
|
||||
register_memory_commands(app, lambda: get_settings())
|
||||
|
||||
|
||||
@app.command()
|
||||
@@ -83,15 +83,6 @@ def ingest(path: Path, cartridge_name: str | None = None) -> None:
|
||||
typer.echo(result.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def _moment(value: str, option: str) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{option} {value!r} is not an ISO timestamp: {error}") from error
|
||||
|
||||
|
||||
@app.command()
|
||||
def search(
|
||||
query: str,
|
||||
@@ -114,8 +105,8 @@ def search(
|
||||
holder=holder or None,
|
||||
scenario=scenario or None,
|
||||
source_id=source_id or None,
|
||||
since=_moment(since, "--since"),
|
||||
until=_moment(until, "--until"),
|
||||
since=moment(since, "--since"),
|
||||
until=moment(until, "--until"),
|
||||
min_trust=None if min_trust < 0 else min_trust,
|
||||
),
|
||||
cursor=cursor or None,
|
||||
@@ -124,60 +115,6 @@ def search(
|
||||
typer.echo(result.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def remember(
|
||||
text: str,
|
||||
holder: str = typer.Option("", "--holder", help="who owns this memory"),
|
||||
origin: MemoryOrigin = MemoryOrigin.self_,
|
||||
scenario: str = "",
|
||||
event_time: str = typer.Option("", "--event-time", help="when it was true, ISO with offset"),
|
||||
precision: TimePrecision = TimePrecision.unknown,
|
||||
retention_until: str = typer.Option("", "--retention-until"),
|
||||
) -> None:
|
||||
settings = get_settings()
|
||||
atom = MemoryService(settings).remember(
|
||||
text,
|
||||
holder_id=holder or settings.default_holder,
|
||||
origin=origin,
|
||||
scenario=scenario,
|
||||
event_time=_moment(event_time, "--event-time"),
|
||||
event_time_precision=precision,
|
||||
retention_until=_moment(retention_until, "--retention-until"),
|
||||
)
|
||||
typer.echo(atom.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def supersede(
|
||||
atom_id: str,
|
||||
text: str,
|
||||
holder: str = typer.Option("", "--holder"),
|
||||
scenario: str = "",
|
||||
event_time: str = typer.Option("", "--event-time"),
|
||||
precision: TimePrecision = TimePrecision.unknown,
|
||||
) -> None:
|
||||
settings = get_settings()
|
||||
atom = MemoryService(settings).supersede(
|
||||
atom_id,
|
||||
text=text,
|
||||
holder_id=holder or settings.default_holder,
|
||||
scenario=scenario,
|
||||
event_time=_moment(event_time, "--event-time"),
|
||||
event_time_precision=precision,
|
||||
)
|
||||
typer.echo(atom.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command()
|
||||
def forget(atom_id: str = "", source_id: str = "", reason: str = "") -> None:
|
||||
report = MemoryService(get_settings()).forget(
|
||||
atom_id=atom_id or None,
|
||||
source_id=source_id or None,
|
||||
reason=reason,
|
||||
)
|
||||
typer.echo(report.model_dump_json(indent=2))
|
||||
|
||||
|
||||
@app.command("list-cartridges")
|
||||
def list_cartridges() -> None:
|
||||
for manifest in CartridgeService(get_settings()).list_manifests():
|
||||
|
||||
@@ -3,7 +3,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def json_dump(value: object) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def moment(value: str, option: str) -> datetime | None:
|
||||
"""An ISO timestamp from the command line, with the option named in the error."""
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{option} {value!r} is not an ISO timestamp: {error}") from error
|
||||
@@ -0,0 +1,130 @@
|
||||
"""CLI face for memory: write it, correct it, erase it, and walk the tiers it lives in.
|
||||
|
||||
Registered onto the main Typer app so the command names stay flat (`ely-eye remember`,
|
||||
`ely-eye archive`) while the memory surface stays in one file. Every command answers in JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import typer
|
||||
|
||||
from .cli_common import json_dump, moment
|
||||
from .config import Settings
|
||||
from .memory import MemoryService
|
||||
from .schemas import MemoryOrigin, TimePrecision
|
||||
|
||||
|
||||
def register(app: typer.Typer, settings: Callable[[], Settings]) -> None:
|
||||
"""Add the memory commands. The settings thunk is passed so one patch in a test redirects
|
||||
every command, the same way the transactive and maintenance faces do."""
|
||||
|
||||
def writer() -> MemoryService:
|
||||
return MemoryService(settings())
|
||||
|
||||
def principal(holder: str) -> str:
|
||||
"""An empty --holder means the local principal, the identity the server writes as."""
|
||||
return holder or settings().default_holder
|
||||
|
||||
@app.command()
|
||||
def remember(
|
||||
text: str,
|
||||
holder: str = typer.Option("", "--holder", help="who owns this memory"),
|
||||
origin: MemoryOrigin = MemoryOrigin.self_,
|
||||
scenario: str = "",
|
||||
event_time: str = typer.Option("", "--event-time", help="when it was true, ISO with offset"),
|
||||
precision: TimePrecision = TimePrecision.unknown,
|
||||
retention_until: str = typer.Option("", "--retention-until"),
|
||||
session: str = typer.Option("", "--session", help="the session this memory belongs to"),
|
||||
importance: float = typer.Option(-1.0, "--importance", help="0..1; below 0 leaves it unset"),
|
||||
) -> None:
|
||||
"""Store one statement as a memory. Repeating it as the same holder updates it."""
|
||||
atom = writer().remember(
|
||||
text,
|
||||
holder_id=principal(holder),
|
||||
origin=origin,
|
||||
scenario=scenario,
|
||||
event_time=moment(event_time, "--event-time"),
|
||||
event_time_precision=precision,
|
||||
retention_until=moment(retention_until, "--retention-until"),
|
||||
session_id=session,
|
||||
importance=None if importance < 0 else importance,
|
||||
)
|
||||
typer.echo(atom.model_dump_json(indent=2))
|
||||
|
||||
@app.command()
|
||||
def supersede(
|
||||
atom_id: str,
|
||||
text: str,
|
||||
holder: str = typer.Option("", "--holder"),
|
||||
scenario: str = "",
|
||||
event_time: str = typer.Option("", "--event-time"),
|
||||
precision: TimePrecision = TimePrecision.unknown,
|
||||
session: str = typer.Option("", "--session"),
|
||||
) -> None:
|
||||
"""Close a memory and record this one as its replacement."""
|
||||
atom = writer().supersede(
|
||||
atom_id,
|
||||
text=text,
|
||||
holder_id=principal(holder),
|
||||
scenario=scenario,
|
||||
event_time=moment(event_time, "--event-time"),
|
||||
event_time_precision=precision,
|
||||
session_id=session,
|
||||
)
|
||||
typer.echo(atom.model_dump_json(indent=2))
|
||||
|
||||
@app.command()
|
||||
def forget(atom_id: str = "", source_id: str = "", reason: str = "") -> None:
|
||||
"""Delete a memory and everything derived from it: vectors, cache, edges."""
|
||||
report = writer().forget(atom_id=atom_id or None, source_id=source_id or None, reason=reason)
|
||||
typer.echo(report.model_dump_json(indent=2))
|
||||
|
||||
@app.command()
|
||||
def working(holder: str, session: str) -> None:
|
||||
"""What one session recorded, in order, with no search involved."""
|
||||
atoms = writer().working_set(holder, session)
|
||||
typer.echo(json_dump([atom.model_dump(mode="json") for atom in atoms]))
|
||||
|
||||
@app.command()
|
||||
def consolidate(scenario: str = "", holder: str = typer.Option("", "--holder")) -> None:
|
||||
"""Roll a lane's new memories into its standing summary and move the watermark."""
|
||||
record = writer().consolidate(principal(holder), scenario)
|
||||
typer.echo("nothing new to roll up" if record is None else record.model_dump_json(indent=2))
|
||||
|
||||
@app.command()
|
||||
def archive(
|
||||
older_than_days: int = typer.Option(90, "--older-than-days"),
|
||||
max_importance: float = typer.Option(-1.0, "--max-importance", help="archive at or under this"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="name the memories without moving them"),
|
||||
) -> None:
|
||||
"""Move never-used, never-valued memories out of the current tier."""
|
||||
report = writer().archive_stale(
|
||||
older_than_days=older_than_days,
|
||||
max_importance=None if max_importance < 0 else max_importance,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
typer.echo(report.model_dump_json(indent=2))
|
||||
|
||||
@app.command()
|
||||
def archived(holder: str = typer.Option("", "--holder")) -> None:
|
||||
"""The long-term tier: what the current view is hiding."""
|
||||
atoms = writer().archived(holder or None)
|
||||
typer.echo(json_dump([atom.model_dump(mode="json") for atom in atoms]))
|
||||
|
||||
@app.command("memory-restore")
|
||||
def restore(atom_ids: list[str]) -> None:
|
||||
"""Bring archived memories back. Named apart from `restore`, which brings the store back."""
|
||||
typer.echo(json_dump({"restored": writer().restore(atom_ids)}))
|
||||
|
||||
@app.command()
|
||||
def conflicts(holder: str = typer.Option("", "--holder")) -> None:
|
||||
"""Every disagreement the store noticed and nobody has settled."""
|
||||
rows = writer().conflicts(holder or None)
|
||||
typer.echo(json_dump([row.model_dump(mode="json") for row in rows]))
|
||||
|
||||
@app.command("resolve-conflicts")
|
||||
def resolve_conflicts(conflict_ids: list[int]) -> None:
|
||||
"""Settle disagreements so retrieval stops choosing a side for the reader."""
|
||||
typer.echo(json_dump({"resolved": writer().resolve_conflicts(conflict_ids)}))
|
||||
@@ -82,9 +82,9 @@ class Database:
|
||||
token_equivalent, metadata_json, holder_id, origin, scenario,
|
||||
event_time, event_time_end, event_time_precision, event_time_source,
|
||||
valid_from, valid_to, supersedes_json, visibility_json, retention_until,
|
||||
quarantined
|
||||
quarantined, session_id, importance, recall_count, last_recalled_at, archived_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(atom_id) DO UPDATE SET
|
||||
modality=excluded.modality,
|
||||
source=excluded.source,
|
||||
@@ -109,7 +109,11 @@ class Database:
|
||||
supersedes_json=excluded.supersedes_json,
|
||||
visibility_json=excluded.visibility_json,
|
||||
retention_until=excluded.retention_until,
|
||||
quarantined=excluded.quarantined
|
||||
quarantined=excluded.quarantined,
|
||||
session_id=excluded.session_id,
|
||||
importance=excluded.importance
|
||||
-- recall_count, last_recalled_at and archived_at stay as the store has them: a
|
||||
-- re-ingested copy of the same atom cannot erase how often it was used.
|
||||
""",
|
||||
[atom_params(atom) for atom in atoms],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
"""The memory tiers, kept out of the query layer the same way the search page is.
|
||||
|
||||
Working memory is what one session recorded. The ledger says what has already been rolled up.
|
||||
Archived rows sit outside the current view on purpose. Recall counts are what the store observed,
|
||||
not what a caller claimed, so they are written here and nowhere else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
|
||||
from .db import Database
|
||||
from .row_codec import from_iso, from_json, row_to_atom, to_json
|
||||
from .schemas import ConflictRecord, ConsolidationRecord, EvidenceAtom
|
||||
|
||||
|
||||
def _moment(value: str) -> datetime:
|
||||
"""A ledger timestamp is never null, so a row whose date will not parse is corrupt, not empty."""
|
||||
moment = from_iso(value)
|
||||
if moment is None:
|
||||
raise ValueError(f"stored timestamp {value!r} is unreadable")
|
||||
return moment
|
||||
|
||||
|
||||
def touch_recalls(database: Database, atom_ids: list[str], at: datetime) -> None:
|
||||
"""Record that a search returned these memories, so use is a fact the store holds."""
|
||||
if not atom_ids:
|
||||
return
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE atoms
|
||||
SET recall_count = recall_count + 1, last_recalled_at = ?
|
||||
WHERE atom_id IN (SELECT value FROM json_each(?))
|
||||
""",
|
||||
(at.isoformat(), to_json(atom_ids)),
|
||||
)
|
||||
|
||||
|
||||
def session_atoms(database: Database, holder_id: str, session_id: str) -> list[EvidenceAtom]:
|
||||
"""What this session recorded and nothing else: the short-term tier, read without searching."""
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM v_atoms_current WHERE holder_id = ? AND session_id = ? ORDER BY ingested_at, atom_id",
|
||||
(holder_id, session_id),
|
||||
).fetchall()
|
||||
return [row_to_atom(row) for row in rows]
|
||||
|
||||
|
||||
def lane_atoms(
|
||||
database: Database, holder_id: str, scenario: str, *, after: datetime | None = None
|
||||
) -> list[EvidenceAtom]:
|
||||
"""Current memories of one holder and scenario, oldest first, from a watermark onward."""
|
||||
watermark = "" if after is None else after.isoformat()
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT * FROM v_atoms_current
|
||||
WHERE holder_id = ? AND scenario = ? AND (? = '' OR ingested_at > ?)
|
||||
ORDER BY ingested_at, atom_id
|
||||
""",
|
||||
(holder_id, scenario, watermark, watermark),
|
||||
).fetchall()
|
||||
return [row_to_atom(row) for row in rows]
|
||||
|
||||
|
||||
def read_ledger(database: Database, holder_id: str, scenario: str) -> ConsolidationRecord | None:
|
||||
with database.connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM consolidations WHERE holder_id = ? AND scenario = ?", (holder_id, scenario)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return ConsolidationRecord(
|
||||
holder_id=row["holder_id"],
|
||||
scenario=row["scenario"],
|
||||
watermark_at=_moment(row["watermark_at"]),
|
||||
rollup_atom_id=row["rollup_atom_id"],
|
||||
atom_count=int(row["atom_count"]),
|
||||
token_equivalent=int(row["token_equivalent"]),
|
||||
consolidated_at=_moment(row["consolidated_at"]),
|
||||
)
|
||||
|
||||
|
||||
def write_ledger(database: Database, record: ConsolidationRecord) -> None:
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO consolidations (
|
||||
holder_id, scenario, watermark_at, rollup_atom_id, atom_count, token_equivalent, consolidated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(holder_id, scenario) DO UPDATE SET
|
||||
watermark_at=excluded.watermark_at,
|
||||
rollup_atom_id=excluded.rollup_atom_id,
|
||||
atom_count=excluded.atom_count,
|
||||
token_equivalent=excluded.token_equivalent,
|
||||
consolidated_at=excluded.consolidated_at
|
||||
""",
|
||||
(
|
||||
record.holder_id,
|
||||
record.scenario,
|
||||
record.watermark_at.isoformat(),
|
||||
record.rollup_atom_id,
|
||||
record.atom_count,
|
||||
record.token_equivalent,
|
||||
record.consolidated_at.isoformat(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def record_conflict(
|
||||
database: Database,
|
||||
*,
|
||||
older_atom_id: str,
|
||||
newer_atom_id: str,
|
||||
holder_id: str,
|
||||
scenario: str,
|
||||
terms: list[str],
|
||||
older_value: str,
|
||||
newer_value: str,
|
||||
detected_at: datetime,
|
||||
) -> ConflictRecord | None:
|
||||
"""Note that two memories cannot both be current. A repeat of the same pair is not a second conflict."""
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO conflicts (
|
||||
older_atom_id, newer_atom_id, holder_id, scenario, terms_json,
|
||||
older_value, newer_value, detected_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
older_atom_id,
|
||||
newer_atom_id,
|
||||
holder_id,
|
||||
scenario,
|
||||
to_json(terms),
|
||||
older_value,
|
||||
newer_value,
|
||||
detected_at.isoformat(),
|
||||
),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT * FROM conflicts WHERE older_atom_id = ? AND newer_atom_id = ?",
|
||||
(older_atom_id, newer_atom_id),
|
||||
).fetchone()
|
||||
return _row_to_conflict(row)
|
||||
|
||||
|
||||
def open_conflicts(database: Database, holder_id: str | None = None) -> list[ConflictRecord]:
|
||||
"""Every disagreement nobody has picked a side in, newest detection first."""
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT * FROM conflicts
|
||||
WHERE resolved_at IS NULL AND (? = '' OR holder_id = ?)
|
||||
ORDER BY detected_at DESC, conflict_id
|
||||
""",
|
||||
(holder_id or "", holder_id or ""),
|
||||
).fetchall()
|
||||
return [_row_to_conflict(row) for row in rows]
|
||||
|
||||
|
||||
def losing_sides(database: Database, atom_ids: list[str]) -> dict[str, str]:
|
||||
"""Among these atoms, which lost to a newer one in a conflict nobody resolved."""
|
||||
if not atom_ids:
|
||||
return {}
|
||||
payload = to_json(atom_ids)
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT older_atom_id, newer_atom_id FROM conflicts
|
||||
WHERE resolved_at IS NULL
|
||||
AND older_atom_id IN (SELECT value FROM json_each(?))
|
||||
AND newer_atom_id IN (SELECT value FROM json_each(?))
|
||||
""",
|
||||
(payload, payload),
|
||||
).fetchall()
|
||||
return {str(row["older_atom_id"]): str(row["newer_atom_id"]) for row in rows}
|
||||
|
||||
|
||||
def resolve_conflicts(database: Database, conflict_ids: list[int], at: datetime) -> int:
|
||||
"""Settle disagreements so retrieval stops suppressing a side on the store's own judgement."""
|
||||
if not conflict_ids:
|
||||
return 0
|
||||
with database.connect() as connection:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
UPDATE conflicts SET resolved_at = ?
|
||||
WHERE resolved_at IS NULL AND conflict_id IN (SELECT value FROM json_each(?))
|
||||
""",
|
||||
(at.isoformat(), to_json([str(conflict_id) for conflict_id in conflict_ids])),
|
||||
)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
|
||||
def archive_atoms(database: Database, atom_ids: list[str], at: datetime) -> int:
|
||||
"""Move memories out of the current tier. The rows stay; only the view stops showing them."""
|
||||
if not atom_ids:
|
||||
return 0
|
||||
with database.connect() as connection:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
UPDATE atoms SET archived_at = ?
|
||||
WHERE archived_at IS NULL AND atom_id IN (SELECT value FROM json_each(?))
|
||||
""",
|
||||
(at.isoformat(), to_json(atom_ids)),
|
||||
)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
|
||||
def restore_atoms(database: Database, atom_ids: list[str]) -> int:
|
||||
if not atom_ids:
|
||||
return 0
|
||||
with database.connect() as connection:
|
||||
cursor = connection.execute(
|
||||
"UPDATE atoms SET archived_at = NULL WHERE atom_id IN (SELECT value FROM json_each(?))",
|
||||
(to_json(atom_ids),),
|
||||
)
|
||||
return int(cursor.rowcount)
|
||||
|
||||
|
||||
def archived_atoms(database: Database, holder_id: str | None = None) -> list[EvidenceAtom]:
|
||||
"""The long-term tier: archived rows, which the current view hides by design."""
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT * FROM atoms
|
||||
WHERE archived_at IS NOT NULL AND (? = '' OR holder_id = ?)
|
||||
ORDER BY archived_at, atom_id
|
||||
""",
|
||||
(holder_id or "", holder_id or ""),
|
||||
).fetchall()
|
||||
return [row_to_atom(row) for row in rows]
|
||||
|
||||
|
||||
def stale_candidates(
|
||||
database: Database, *, before: datetime, max_importance: float | None
|
||||
) -> list[str]:
|
||||
"""Current memories that were ingested before the cutoff, were never returned by a search, and
|
||||
were never marked more important than the floor; an unset importance is not a reason to keep.
|
||||
"""
|
||||
with database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT atom_id FROM v_atoms_current
|
||||
WHERE ingested_at < ? AND recall_count = 0
|
||||
AND (? IS NULL OR importance IS NULL OR importance <= ?)
|
||||
ORDER BY ingested_at, atom_id
|
||||
""",
|
||||
(before.isoformat(), max_importance, max_importance),
|
||||
).fetchall()
|
||||
return [str(row["atom_id"]) for row in rows]
|
||||
|
||||
|
||||
def _row_to_conflict(row: sqlite3.Row) -> ConflictRecord:
|
||||
return ConflictRecord(
|
||||
conflict_id=int(row["conflict_id"]),
|
||||
older_atom_id=row["older_atom_id"],
|
||||
newer_atom_id=row["newer_atom_id"],
|
||||
holder_id=row["holder_id"],
|
||||
scenario=row["scenario"],
|
||||
terms=[str(item) for item in from_json(row["terms_json"], [])],
|
||||
older_value=row["older_value"],
|
||||
newer_value=row["newer_value"],
|
||||
detected_at=_moment(row["detected_at"]),
|
||||
resolved_at=from_iso(row["resolved_at"]),
|
||||
)
|
||||
@@ -9,15 +9,18 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import trust
|
||||
from . import lifecycle_store, trust
|
||||
from .cache_fabric import CacheFabric
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .schemas import (
|
||||
ArchiveReport,
|
||||
ConflictRecord,
|
||||
ConsolidationRecord,
|
||||
EvidenceAtom,
|
||||
ForgetReport,
|
||||
MemoryOrigin,
|
||||
@@ -32,9 +35,12 @@ from .storage import token_equivalent
|
||||
from .vector_index import VectorIndex
|
||||
from .visual_index import VisualIndex
|
||||
|
||||
# Trust scores describe extraction fidelity, not truthfulness: a remembered sentence is stored verbatim, so origin carries the epistemic weight.
|
||||
# A remembered sentence is stored verbatim, so trust scores carry no weight here; origin does.
|
||||
MEMORY_TRUST = TrustScores(parser=1.0, ocr=0.0, model=0.0)
|
||||
|
||||
# The rollup atom's id suffix: one standing summary per holder and scenario, replaced in place.
|
||||
ROLLUP_SUFFIX = ":rollup"
|
||||
|
||||
|
||||
class MemoryService:
|
||||
def __init__(
|
||||
@@ -63,6 +69,8 @@ class MemoryService:
|
||||
supersedes: list[str] | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
retention_until: datetime | None = None,
|
||||
session_id: str = "",
|
||||
importance: float | None = None,
|
||||
) -> EvidenceAtom:
|
||||
"""Store one statement as a memory owned by a holder.
|
||||
|
||||
@@ -116,12 +124,44 @@ class MemoryService:
|
||||
valid_from=valid_from,
|
||||
visibility=visibility or Visibility(),
|
||||
retention_until=retention_until,
|
||||
session_id=session_id,
|
||||
importance=importance,
|
||||
)
|
||||
trust.stamp([atom], origin=origin)
|
||||
self.db.upsert_atoms([atom])
|
||||
self.vectors.sync_atoms([atom.atom_id])
|
||||
self._note_conflicts(atom)
|
||||
return atom
|
||||
|
||||
def _note_conflicts(self, atom: EvidenceAtom) -> None:
|
||||
"""A number that moves inside one holder's scenario is a disagreement, not a second fact.
|
||||
|
||||
Only a memory that replaces nothing can start a conflict: a supersede has already
|
||||
picked a side, and recording it as unresolved would contradict that.
|
||||
"""
|
||||
if atom.supersedes:
|
||||
return
|
||||
stated = trust.numeric_values(atom.text)
|
||||
if not stated:
|
||||
return
|
||||
for earlier in lifecycle_store.lane_atoms(self.db, atom.holder_id, atom.scenario):
|
||||
if earlier.atom_id == atom.atom_id:
|
||||
continue
|
||||
term = trust.conflicting_number(trust.numeric_values(earlier.text), stated)
|
||||
if term is None:
|
||||
continue
|
||||
lifecycle_store.record_conflict(
|
||||
self.db,
|
||||
older_atom_id=earlier.atom_id,
|
||||
newer_atom_id=atom.atom_id,
|
||||
holder_id=atom.holder_id,
|
||||
scenario=atom.scenario,
|
||||
terms=[term],
|
||||
older_value=trust.numeric_values(earlier.text)[term],
|
||||
newer_value=stated[term],
|
||||
detected_at=utc_now(),
|
||||
)
|
||||
|
||||
def supersede(
|
||||
self,
|
||||
atom_id: str,
|
||||
@@ -133,6 +173,8 @@ class MemoryService:
|
||||
event_time: datetime | None = None,
|
||||
event_time_precision: TimePrecision = TimePrecision.unknown,
|
||||
event_time_source: str = "",
|
||||
session_id: str = "",
|
||||
importance: float | None = None,
|
||||
) -> EvidenceAtom:
|
||||
"""Close an old memory and record the new one as its replacement.
|
||||
|
||||
@@ -159,6 +201,8 @@ class MemoryService:
|
||||
event_time_source=event_time_source,
|
||||
supersedes=[atom_id],
|
||||
valid_from=moment,
|
||||
session_id=session_id,
|
||||
importance=importance,
|
||||
)
|
||||
self.db.close_atoms([atom_id], moment)
|
||||
self.vectors.sync_atoms([atom_id])
|
||||
@@ -206,6 +250,144 @@ class MemoryService:
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
def working_set(self, holder_id: str, session_id: str) -> list[EvidenceAtom]:
|
||||
"""What this session recorded, in the order it recorded it: the short-term tier.
|
||||
|
||||
No search is involved. A session asking for its own context gets exactly what it put
|
||||
down, which is the whole point of a working set rather than a wider top-k.
|
||||
"""
|
||||
if not session_id.strip():
|
||||
raise ValueError("a working set needs a session; without one every memory is working memory")
|
||||
return lifecycle_store.session_atoms(self.db, holder_id, session_id)
|
||||
|
||||
def consolidate(
|
||||
self, holder_id: str, scenario: str = "", *, at: datetime | None = None
|
||||
) -> ConsolidationRecord | None:
|
||||
"""Roll a lane's new memories into one standing summary and move the watermark.
|
||||
|
||||
The summary replaces its own previous version under a fixed id, so a lane holds one
|
||||
rollup instead of a stack of them, and the ledger's watermark is what keeps a memory out
|
||||
of the next pass. Nothing is deleted: the individual memories stay current and findable,
|
||||
because a summary that erased the detail could not be questioned about where it came from.
|
||||
"""
|
||||
moment = at or utc_now()
|
||||
ledger = lifecycle_store.read_ledger(self.db, holder_id, scenario)
|
||||
batch = [
|
||||
atom
|
||||
for atom in lifecycle_store.lane_atoms(
|
||||
self.db, holder_id, scenario, after=ledger.watermark_at if ledger else None
|
||||
)
|
||||
if not atom.atom_id.endswith(ROLLUP_SUFFIX)
|
||||
]
|
||||
if not batch:
|
||||
return None
|
||||
|
||||
lines: list[str] = []
|
||||
if ledger:
|
||||
previous = self.db.get_atoms([ledger.rollup_atom_id])
|
||||
if previous:
|
||||
lines.append(previous[0].text)
|
||||
lines.extend(atom.text for atom in batch)
|
||||
body = "\n".join(lines)
|
||||
|
||||
digest = hashlib.sha256(f"{holder_id}\n{scenario}".encode()).hexdigest()
|
||||
source_id = f"rollup-{digest[:16]}"
|
||||
atom_id = f"{source_id}{ROLLUP_SUFFIX}"
|
||||
self.db.upsert_sources(
|
||||
[
|
||||
SourceRecord(
|
||||
source_id=source_id,
|
||||
path=f"memory://{holder_id}/{scenario or 'general'}#rollup",
|
||||
kind="memory",
|
||||
mime="text/plain",
|
||||
sha256=hashlib.sha256(body.encode("utf-8")).hexdigest(),
|
||||
size_bytes=len(body.encode("utf-8")),
|
||||
created_at=moment,
|
||||
metadata={"holder_id": holder_id, "rollup": True},
|
||||
)
|
||||
]
|
||||
)
|
||||
rollup = EvidenceAtom(
|
||||
atom_id=atom_id,
|
||||
modality=Modality.text,
|
||||
source_id=source_id,
|
||||
source=f"memory://{holder_id}/{scenario or 'general'}#rollup",
|
||||
ingested_at=moment,
|
||||
text=body,
|
||||
trust=MEMORY_TRUST,
|
||||
token_equivalent=token_equivalent(body),
|
||||
holder_id=holder_id,
|
||||
origin=MemoryOrigin.self_,
|
||||
scenario=scenario,
|
||||
)
|
||||
self.db.upsert_atoms([rollup])
|
||||
self.vectors.sync_atoms([atom_id])
|
||||
# The watermark is the newest memory actually rolled up, not the clock of the pass: a
|
||||
# consolidation run with a clock behind the memories would otherwise count them twice.
|
||||
record = ConsolidationRecord(
|
||||
holder_id=holder_id,
|
||||
scenario=scenario,
|
||||
watermark_at=max(atom.ingested_at for atom in batch),
|
||||
rollup_atom_id=atom_id,
|
||||
atom_count=(ledger.atom_count if ledger else 0) + len(batch),
|
||||
token_equivalent=token_equivalent(body),
|
||||
consolidated_at=moment,
|
||||
)
|
||||
lifecycle_store.write_ledger(self.db, record)
|
||||
return record
|
||||
|
||||
def archive_stale(
|
||||
self,
|
||||
*,
|
||||
older_than_days: int,
|
||||
max_importance: float | None = None,
|
||||
at: datetime | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> ArchiveReport:
|
||||
"""Move memories that were never used and never valued out of the current tier.
|
||||
|
||||
The rule is stated instead of hidden in a decay formula: a memory goes when it was
|
||||
recorded before the cutoff, no search ever returned it, and nobody marked it important.
|
||||
Archived rows can be brought back; deleting them would make a wrong judgement
|
||||
unrecoverable, and the store's own estimate of what mattered is exactly the kind of thing
|
||||
that turns out to be wrong.
|
||||
"""
|
||||
if older_than_days < 0:
|
||||
raise ValueError(f"older_than_days must not be negative, got {older_than_days}")
|
||||
moment = at or utc_now()
|
||||
cutoff = moment - timedelta(days=older_than_days)
|
||||
candidates = lifecycle_store.stale_candidates(self.db, before=cutoff, max_importance=max_importance)
|
||||
moved = 0 if dry_run else lifecycle_store.archive_atoms(self.db, candidates, moment)
|
||||
if not dry_run and moved != len(candidates):
|
||||
raise ValueError(f"{len(candidates)} memories were chosen for the archive but {moved} were moved")
|
||||
floor = "any importance" if max_importance is None else f"importance at or under {max_importance}"
|
||||
return ArchiveReport(
|
||||
atom_ids=candidates,
|
||||
archived_at=moment,
|
||||
rule=f"recorded before {cutoff.isoformat()}, never returned by a search, {floor}",
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
def archived(self, holder_id: str | None = None) -> list[EvidenceAtom]:
|
||||
"""The long-term tier: what the current view hides, listed rather than guessed at."""
|
||||
return lifecycle_store.archived_atoms(self.db, holder_id)
|
||||
|
||||
def restore(self, atom_ids: list[str]) -> int:
|
||||
"""Bring archived memories back: the archive is a tier, not a verdict."""
|
||||
if not atom_ids:
|
||||
raise ValueError("restore needs atom ids; nothing was named")
|
||||
return lifecycle_store.restore_atoms(self.db, atom_ids)
|
||||
|
||||
def conflicts(self, holder_id: str | None = None) -> list[ConflictRecord]:
|
||||
"""Every disagreement the store noticed and nobody has settled."""
|
||||
return lifecycle_store.open_conflicts(self.db, holder_id)
|
||||
|
||||
def resolve_conflicts(self, conflict_ids: list[int], *, at: datetime | None = None) -> int:
|
||||
"""Settle disagreements so retrieval stops dropping a side on the store's own reading."""
|
||||
if not conflict_ids:
|
||||
raise ValueError("resolving needs conflict ids; nothing was named")
|
||||
return lifecycle_store.resolve_conflicts(self.db, conflict_ids, at or utc_now())
|
||||
|
||||
def _drop_derived_vectors(self, atom_ids: list[str]) -> tuple[int, bool]:
|
||||
"""A forgotten memory must stop answering from the derived indexes too.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import sqlite3
|
||||
from .storage import token_equivalent, visual_token_equivalent
|
||||
from .text_index import index_text
|
||||
|
||||
SCHEMA_VERSION = 9
|
||||
SCHEMA_VERSION = 10
|
||||
|
||||
ATOM_COLUMNS_V2 = """
|
||||
atom_id TEXT PRIMARY KEY,
|
||||
@@ -100,6 +100,37 @@ _BASE_TABLES = """
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- What has already been folded into a rollup, one row per holder and scenario. The
|
||||
-- watermark is what makes consolidation replayable: the next pass starts where the last one
|
||||
-- stopped, so the same memory is never counted into two rollups.
|
||||
CREATE TABLE IF NOT EXISTS consolidations (
|
||||
holder_id TEXT NOT NULL,
|
||||
scenario TEXT NOT NULL DEFAULT '',
|
||||
watermark_at TEXT NOT NULL,
|
||||
rollup_atom_id TEXT NOT NULL REFERENCES atoms(atom_id) ON DELETE CASCADE,
|
||||
atom_count INTEGER NOT NULL,
|
||||
token_equivalent INTEGER NOT NULL,
|
||||
consolidated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (holder_id, scenario)
|
||||
);
|
||||
|
||||
-- Two memories that cannot both be current. The store records the disagreement even when no
|
||||
-- person has picked a winner; retrieval reads this table to keep the newer side instead of
|
||||
-- packing both numbers into the same prompt.
|
||||
CREATE TABLE IF NOT EXISTS conflicts (
|
||||
conflict_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
older_atom_id TEXT NOT NULL REFERENCES atoms(atom_id) ON DELETE CASCADE,
|
||||
newer_atom_id TEXT NOT NULL REFERENCES atoms(atom_id) ON DELETE CASCADE,
|
||||
holder_id TEXT NOT NULL,
|
||||
scenario TEXT NOT NULL DEFAULT '',
|
||||
terms_json TEXT NOT NULL,
|
||||
older_value TEXT NOT NULL,
|
||||
newer_value TEXT NOT NULL,
|
||||
detected_at TEXT NOT NULL,
|
||||
resolved_at TEXT,
|
||||
UNIQUE (older_atom_id, newer_atom_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cache_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts TEXT NOT NULL,
|
||||
@@ -151,11 +182,12 @@ tokenize = 'unicode61'
|
||||
-- per CJK character and per identifier fragment, which is what makes those queries match.
|
||||
"""
|
||||
|
||||
# A memory is usable now when nothing closed it, it had begun by now, and its retention has not lapsed; unknown bounds read as visible, because silence about when a legacy row was true is not evidence that it is false.
|
||||
# A memory is usable now when nothing closed it, it had begun by now, its retention has not lapsed, and nobody archived it away from the current tier; unknown bounds read as visible, because silence about when a legacy row was true is not evidence that it is false.
|
||||
VIEWS = """
|
||||
CREATE VIEW IF NOT EXISTS v_atoms_current AS
|
||||
SELECT * FROM atoms
|
||||
WHERE valid_to IS NULL
|
||||
AND archived_at IS NULL
|
||||
AND (valid_from IS NULL OR valid_from <= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
|
||||
AND (retention_until IS NULL OR retention_until >= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'));
|
||||
|
||||
@@ -164,6 +196,8 @@ VIEWS = """
|
||||
CREATE INDEX IF NOT EXISTS idx_atoms_holder ON atoms(holder_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_atoms_scenario ON atoms(scenario);
|
||||
CREATE INDEX IF NOT EXISTS idx_atoms_valid_to ON atoms(valid_to);
|
||||
CREATE INDEX IF NOT EXISTS idx_atoms_session ON atoms(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_atoms_archived ON atoms(archived_at);
|
||||
"""
|
||||
|
||||
# Columns the first version lacked, in the order added; each is nullable or defaulted so an existing row keeps its meaning.
|
||||
@@ -185,6 +219,16 @@ _V1_TO_V2_COLUMNS = (
|
||||
# Columns added after the second version, grouped by the schema version that introduced them.
|
||||
_NEW_ATOM_COLUMNS: tuple[tuple[int, tuple[tuple[str, str], ...]], ...] = (
|
||||
(9, (("quarantined", "INTEGER NOT NULL DEFAULT 0"),)),
|
||||
(
|
||||
10,
|
||||
(
|
||||
("session_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
("importance", "REAL"),
|
||||
("recall_count", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("last_recalled_at", "TEXT"),
|
||||
("archived_at", "TEXT"),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -270,6 +314,10 @@ def apply(connection: sqlite3.Connection) -> None:
|
||||
for name, definition in columns:
|
||||
if name not in existing:
|
||||
connection.execute(f"ALTER TABLE atoms ADD COLUMN {name} {definition}")
|
||||
if version < 10:
|
||||
# CREATE VIEW IF NOT EXISTS keeps the old body, and the tenth version changed which rows
|
||||
# the current tier contains, so an upgraded database has to be told to look again.
|
||||
connection.execute("DROP VIEW IF EXISTS v_atoms_current")
|
||||
|
||||
connection.executescript(VIEWS)
|
||||
connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
||||
|
||||
@@ -14,11 +14,11 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import cartridge_store
|
||||
from . import cartridge_store, lifecycle_store
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .embeddings import EmbeddingService
|
||||
from .schemas import EvidenceAtom, RetrievalHit, SearchFilters, SearchResult
|
||||
from .schemas import EvidenceAtom, RetrievalHit, SearchFilters, SearchResult, utc_now
|
||||
from .store_search import FtsPage, search_atoms
|
||||
from .vector_index import VectorIndex
|
||||
from .visual_index import VisualIndex
|
||||
@@ -89,7 +89,8 @@ class RetrievalService:
|
||||
rankings.append(list(visual))
|
||||
for atom_id, cosine in visual.items():
|
||||
scores[atom_id]["visual_score"] = cosine
|
||||
hits = self._rank(rankings, scores, limit)
|
||||
hits = self._settle_conflicts(self._rank(rankings, scores, limit * CANDIDATE_WINDOW))[:limit]
|
||||
lifecycle_store.touch_recalls(self.db, [hit.atom.atom_id for hit in hits], utc_now())
|
||||
more = page.has_more or len(rankings[0]) > limit
|
||||
return SearchResult(
|
||||
query=query,
|
||||
@@ -179,6 +180,18 @@ class RetrievalService:
|
||||
)
|
||||
return hits
|
||||
|
||||
def _settle_conflicts(self, hits: list[RetrievalHit]) -> list[RetrievalHit]:
|
||||
"""Drop the older side of a disagreement nobody settled, so one page cannot contradict itself.
|
||||
|
||||
Both memories stay in the store and this only filters the page; the conflict row names
|
||||
which side was dropped and why. Settling is the holder's call, and until it happens the
|
||||
newer statement is what a reader should be shown.
|
||||
"""
|
||||
losers = lifecycle_store.losing_sides(self.db, [hit.atom.atom_id for hit in hits])
|
||||
if not losers:
|
||||
return hits
|
||||
return [hit for hit in hits if hit.atom.atom_id not in losers]
|
||||
|
||||
def recall(self, query: str, top_k: int | None = None, *, filters: SearchFilters | None = None) -> list[EvidenceAtom]:
|
||||
"""The atoms a search would put in front of a model, no scoring attached."""
|
||||
return [hit.atom for hit in self.search(query, top_k, filters=filters).hits]
|
||||
|
||||
@@ -70,6 +70,11 @@ def atom_params(atom: EvidenceAtom) -> tuple[Any, ...]:
|
||||
to_json(atom.visibility.model_dump()),
|
||||
atom.retention_until.isoformat() if atom.retention_until else None,
|
||||
1 if atom.quarantined else 0,
|
||||
atom.session_id,
|
||||
atom.importance,
|
||||
atom.recall_count,
|
||||
atom.last_recalled_at.isoformat() if atom.last_recalled_at else None,
|
||||
atom.archived_at.isoformat() if atom.archived_at else None,
|
||||
)
|
||||
|
||||
|
||||
@@ -102,6 +107,11 @@ def row_to_atom(row: sqlite3.Row) -> EvidenceAtom:
|
||||
visibility=Visibility(**from_json(row["visibility_json"], {})),
|
||||
retention_until=from_iso(row["retention_until"]),
|
||||
quarantined=bool(row["quarantined"]),
|
||||
session_id=row["session_id"],
|
||||
importance=None if row["importance"] is None else float(row["importance"]),
|
||||
recall_count=int(row["recall_count"]),
|
||||
last_recalled_at=from_iso(row["last_recalled_at"]),
|
||||
archived_at=from_iso(row["archived_at"]),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -109,6 +109,11 @@ class EvidenceAtom(BaseModel):
|
||||
visibility: Visibility = Field(default_factory=Visibility)
|
||||
retention_until: datetime | None = None
|
||||
quarantined: bool = False
|
||||
session_id: str = ""
|
||||
importance: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
recall_count: int = Field(default=0, ge=0)
|
||||
last_recalled_at: datetime | None = None
|
||||
archived_at: datetime | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _times_are_ordered(self) -> EvidenceAtom:
|
||||
@@ -120,7 +125,7 @@ class EvidenceAtom(BaseModel):
|
||||
raise ValueError(f"{self.atom_id}: an atom cannot supersede itself")
|
||||
return self
|
||||
|
||||
@field_validator("ingested_at", "event_time", "event_time_end", "valid_from", "valid_to", "retention_until")
|
||||
@field_validator("ingested_at", "event_time", "event_time_end", "valid_from", "valid_to", "retention_until", "last_recalled_at", "archived_at")
|
||||
@classmethod
|
||||
def _times_are_aware(cls, value: datetime | None) -> datetime | None:
|
||||
if value is not None and value.tzinfo is None:
|
||||
@@ -281,6 +286,48 @@ class ForgetReport(BaseModel):
|
||||
reason: str
|
||||
|
||||
|
||||
class ConsolidationRecord(BaseModel):
|
||||
"""One lane of the long-term ledger: everything up to the watermark has been rolled up, and
|
||||
the next pass starts there so no memory is counted into two rollups.
|
||||
"""
|
||||
|
||||
holder_id: str
|
||||
scenario: str = ""
|
||||
watermark_at: datetime
|
||||
rollup_atom_id: str
|
||||
atom_count: int = Field(ge=0)
|
||||
token_equivalent: int = Field(ge=0)
|
||||
consolidated_at: datetime
|
||||
|
||||
|
||||
class ConflictRecord(BaseModel):
|
||||
"""Two memories of one holder that cannot both be current, with the value each side states."""
|
||||
|
||||
conflict_id: int
|
||||
older_atom_id: str
|
||||
newer_atom_id: str
|
||||
holder_id: str
|
||||
scenario: str = ""
|
||||
terms: list[str] = Field(default_factory=list)
|
||||
older_value: str
|
||||
newer_value: str
|
||||
detected_at: datetime
|
||||
resolved_at: datetime | None = None
|
||||
|
||||
|
||||
class ArchiveReport(BaseModel):
|
||||
"""Which memories left the current tier, and the rule that moved them.
|
||||
|
||||
`archived_at` is the moment the rule stamps; on a dry run it is the moment it would
|
||||
stamp, and `atom_ids` is the list the sentence would fall on.
|
||||
"""
|
||||
|
||||
atom_ids: list[str]
|
||||
archived_at: datetime
|
||||
rule: str
|
||||
dry_run: bool = False
|
||||
|
||||
|
||||
class RememberRequest(BaseModel):
|
||||
"""A memory as a caller states it. Holder, origin and ingestion time are stamped by the
|
||||
server, so they are absent here on purpose: a client cannot claim to be somebody else.
|
||||
@@ -292,6 +339,8 @@ class RememberRequest(BaseModel):
|
||||
event_time_precision: TimePrecision = TimePrecision.unknown
|
||||
event_time_source: str = ""
|
||||
retention_until: datetime | None = None
|
||||
session_id: str = ""
|
||||
importance: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
|
||||
|
||||
class SupersedeRequest(RememberRequest):
|
||||
|
||||
@@ -8,6 +8,7 @@ second, non-quarantined source says it too.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
from unicodedata import normalize
|
||||
@@ -192,3 +193,72 @@ def _spread(pairs: list[str]) -> list[str]:
|
||||
if len(pairs) <= CORROBORATION_TERMS:
|
||||
return pairs
|
||||
return [pairs[0], pairs[len(pairs) // 2], pairs[-1]]
|
||||
|
||||
|
||||
def numeric_values(text: str) -> dict[str, str]:
|
||||
"""Every number a memory states, keyed by what it measures, so a moving value can be named.
|
||||
|
||||
A number is attributed twice: to the subject the memory leads with, and to the content word
|
||||
standing beside it. 「恒温器放在书房,设定 22 度」 and 「恒温器调到 24 度」 phrase the setting
|
||||
differently but lead with the same thing, and it is that shared label which makes the pair one
|
||||
disagreement about one quantity rather than two unrelated counts.
|
||||
"""
|
||||
try:
|
||||
tokens = fts_terms(text)
|
||||
except ValueError:
|
||||
return {}
|
||||
subject = _label(tokens[0], first=True) if tokens else ""
|
||||
values: dict[str, str] = {}
|
||||
beside = ""
|
||||
for token in tokens:
|
||||
stated = _NUMBER.findall(token)
|
||||
if not stated:
|
||||
beside = _label(token) or beside
|
||||
continue
|
||||
for value in stated:
|
||||
if subject:
|
||||
values.setdefault(subject, value)
|
||||
if beside:
|
||||
values.setdefault(beside, value)
|
||||
return values
|
||||
|
||||
|
||||
def conflicting_number(older: dict[str, str], newer: dict[str, str]) -> str | None:
|
||||
"""The one quantity both memories state a different value for, or None if they do not disagree.
|
||||
|
||||
Only a shared label counts: two memories that each mention a number are not in conflict until
|
||||
they name the same thing. The label is reported, not the verdict — which side is current is a
|
||||
decision for the holder, and retrieval only drops the older side while the disagreement stands.
|
||||
"""
|
||||
for label in sorted(set(older) & set(newer)):
|
||||
if _as_float(older[label]) != _as_float(newer[label]):
|
||||
return label
|
||||
return None
|
||||
|
||||
|
||||
def _as_float(value: str) -> float:
|
||||
# The regex only accepts numbers, so a value that will not parse means the pattern changed.
|
||||
return float(value)
|
||||
|
||||
|
||||
def _label(token: str, *, first: bool = False) -> str:
|
||||
"""The content word a number hangs on: a CJK run's two-character tail, or a plain Latin word.
|
||||
|
||||
The subject of a memory is read from the head of its first run instead, because two memories
|
||||
about the same thing usually phrase the rest differently: 「恒温器放在书房,设定 22 度」 and
|
||||
「恒温器调到 24 度」 share only their subject, and that is the label which makes them disagree.
|
||||
"""
|
||||
letters = "".join(char for char in token if is_cjk(char))
|
||||
if letters:
|
||||
pairs = cjk_pairs(letters)
|
||||
if not pairs:
|
||||
return ""
|
||||
return pairs[0] if first else pairs[-1]
|
||||
word = token.strip(_PUNCTUATION).lower()
|
||||
if not word or word in STOPWORDS or any(char.isdigit() for char in word):
|
||||
return ""
|
||||
return word
|
||||
|
||||
|
||||
_PUNCTUATION = " \t\n.,;:!?。,;:!?、"
|
||||
_NUMBER = re.compile(r"[-+]?\d+(?:\.\d+)?")
|
||||
@@ -231,6 +231,7 @@ class VectorIndex:
|
||||
"""
|
||||
SELECT atom_id FROM atoms
|
||||
WHERE atom_id IN (SELECT value FROM json_each(?))
|
||||
AND archived_at IS NULL
|
||||
AND (valid_to IS NULL OR valid_to > strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
|
||||
AND (valid_from IS NULL OR valid_from <= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
|
||||
AND (retention_until IS NULL OR retention_until >= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
|
||||
|
||||
@@ -67,3 +67,31 @@ def test_a_forget_that_names_nothing_is_rejected(client: TestClient) -> None:
|
||||
response = client.post("/api/memory/forget", json={"reason": "vague"})
|
||||
assert response.status_code == 422
|
||||
assert "exactly one" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_a_session_survives_the_http_face(client: TestClient) -> None:
|
||||
first = client.post("/api/memory", json={"text": "烤箱要除垢", "session_id": "cooking"}).json()
|
||||
second = client.post("/api/memory", json={"text": "面粉在储物架第二层", "session_id": "cooking"}).json()
|
||||
client.post("/api/memory", json={"text": "周报周五交"})
|
||||
|
||||
working = client.get("/api/memory/working", params={"holder": "wife", "session": "cooking"})
|
||||
|
||||
assert working.status_code == 200
|
||||
assert [atom["atom_id"] for atom in working.json()] == [first["atom_id"], second["atom_id"]]
|
||||
|
||||
|
||||
def test_working_refuses_a_session_it_cannot_name(client: TestClient) -> None:
|
||||
response = client.get("/api/memory/working", params={"holder": "wife", "session": ""})
|
||||
assert response.status_code == 422
|
||||
assert "a working set needs a session" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_conflicts_reports_both_sides_of_a_disagreement(client: TestClient) -> None:
|
||||
client.post("/api/memory", json={"text": "恒温器放在书房,设定 22 度", "scenario": "home"})
|
||||
client.post("/api/memory", json={"text": "恒温器调到 24 度", "scenario": "home"})
|
||||
|
||||
listed = client.get("/api/memory/conflicts").json()
|
||||
|
||||
assert [(row["older_value"], row["newer_value"]) for row in listed] == [("22", "24")]
|
||||
assert listed[0]["holder_id"] == "wife"
|
||||
assert client.get("/api/memory/conflicts", params={"holder": "husband"}).json() == []
|
||||
@@ -106,3 +106,55 @@ def test_the_cli_answers_in_json_a_machine_can_read(db: Database) -> None:
|
||||
result = runner.invoke(app, ["remember", "钥匙在玄关抽屉", "--holder", "wife"])
|
||||
assert result.exit_code == 0
|
||||
assert json.loads(result.output)["text"] == "钥匙在玄关抽屉"
|
||||
|
||||
|
||||
def test_working_lists_one_session_in_the_order_it_was_recorded(db: Database) -> None:
|
||||
first = run("remember", "烤箱要除垢", "--session", "cooking")
|
||||
second = run("remember", "面粉在储物架第二层", "--session", "cooking")
|
||||
run("remember", "周报周五交")
|
||||
|
||||
working = run("working", "husband", "cooking")
|
||||
|
||||
assert [atom["atom_id"] for atom in working] == [first["atom_id"], second["atom_id"]]
|
||||
|
||||
|
||||
def test_working_refuses_to_guess_which_session_is_working(db: Database) -> None:
|
||||
result = runner.invoke(app, ["working", "husband", ""])
|
||||
assert result.exit_code != 0
|
||||
assert "a working set needs a session" in str(result.exception)
|
||||
|
||||
|
||||
def test_consolidate_rolls_a_lane_up_once_and_then_says_nothing_new(db: Database) -> None:
|
||||
run("remember", "烤箱要除垢", "--scenario", "home")
|
||||
record = run("consolidate", "--scenario", "home", "--holder", "husband")
|
||||
assert record["atom_count"] == 1 and record["rollup_atom_id"]
|
||||
|
||||
result = runner.invoke(app, ["consolidate", "--scenario", "home", "--holder", "husband"])
|
||||
assert result.exit_code == 0
|
||||
assert result.output.strip() == "nothing new to roll up"
|
||||
|
||||
|
||||
def test_the_archive_can_be_read_before_it_moves_anything(db: Database) -> None:
|
||||
atom = run("remember", "前租户的 wifi 密码")
|
||||
|
||||
preview = run("archive", "--older-than-days", "0", "--dry-run")
|
||||
assert preview["dry_run"] is True and preview["atom_ids"] == [atom["atom_id"]]
|
||||
assert run("archived") == []
|
||||
|
||||
moved = run("archive", "--older-than-days", "0")
|
||||
assert moved["dry_run"] is False and moved["atom_ids"] == [atom["atom_id"]]
|
||||
assert [row["atom_id"] for row in run("archived")] == [atom["atom_id"]]
|
||||
assert run("memory-restore", atom["atom_id"]) == {"restored": 1}
|
||||
assert run("archived") == []
|
||||
|
||||
|
||||
def test_the_cli_shows_and_settles_a_disagreement(db: Database) -> None:
|
||||
run("remember", "恒温器放在书房,设定 22 度", "--scenario", "home")
|
||||
run("remember", "恒温器调到 24 度", "--scenario", "home")
|
||||
|
||||
listed = run("conflicts", "--holder", "husband")
|
||||
assert len(listed) == 1
|
||||
assert listed[0]["older_value"] == "22" and listed[0]["newer_value"] == "24"
|
||||
|
||||
assert run("resolve-conflicts", str(listed[0]["conflict_id"])) == {"resolved": 1}
|
||||
assert run("conflicts", "--holder", "husband") == []
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Two memories cannot both be true: the store notices, answers with the newer side, and waits.
|
||||
|
||||
Settling a disagreement is the holder's act. Until then the store chooses, but it records that
|
||||
it chose, and it keeps both sides so the choice can be undone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.memory import MemoryService
|
||||
from ely_eye.retrieval import RetrievalService
|
||||
from ely_eye.schemas import MemoryOrigin, SearchFilters
|
||||
|
||||
NOW = datetime(2026, 5, 1, 12, 0, tzinfo=UTC)
|
||||
OLD = "恒温器放在书房,设定 22 度"
|
||||
NEW = "恒温器调到 24 度"
|
||||
|
||||
|
||||
def _pair(tmp_path, *, scenario: str = "home", newer_holder: str = "wife") -> tuple[MemoryService, Database, list[str]]:
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
database = Database(settings)
|
||||
database.init()
|
||||
service = MemoryService(settings=settings, database=database)
|
||||
older = service.remember(OLD, holder_id="wife", origin=MemoryOrigin.self_, scenario=scenario).atom_id
|
||||
newer = service.remember(NEW, holder_id=newer_holder, origin=MemoryOrigin.self_, scenario=scenario).atom_id
|
||||
return service, database, [older, newer]
|
||||
|
||||
|
||||
def test_a_number_that_changed_leaves_a_conflict_naming_both_sides(tmp_path) -> None:
|
||||
service, _, ids = _pair(tmp_path)
|
||||
|
||||
clashes = service.conflicts("wife")
|
||||
|
||||
assert len(clashes) == 1
|
||||
record = clashes[0]
|
||||
assert (record.older_atom_id, record.newer_atom_id) == (ids[0], ids[1])
|
||||
assert record.older_value == "22" and record.newer_value == "24"
|
||||
assert record.terms, "the record names what the two memories disagree about"
|
||||
assert record.resolved_at is None
|
||||
|
||||
|
||||
def test_repeating_the_same_statement_adds_no_memory_and_no_conflict(tmp_path) -> None:
|
||||
service, database, _ = _pair(tmp_path)
|
||||
before = len(service.conflicts("wife"))
|
||||
|
||||
service.remember(NEW, holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
|
||||
assert database.atom_count() == 2, "the same holder stating the same sentence holds one memory"
|
||||
assert len(service.conflicts("wife")) == before, (
|
||||
"a repeat cannot manufacture a second disagreement"
|
||||
)
|
||||
|
||||
|
||||
def test_a_memory_that_agrees_leaves_nothing_to_settle(tmp_path) -> None:
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
database = Database(settings)
|
||||
database.init()
|
||||
service = MemoryService(settings=settings, database=database)
|
||||
service.remember(OLD, holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
|
||||
service.remember("恒温器设定 22 度", holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
|
||||
assert service.conflicts("wife") == [], (
|
||||
"restating the same reading in other words is agreement, and the rule must not flinch at it"
|
||||
)
|
||||
|
||||
|
||||
def test_two_holders_who_disagree_are_not_a_conflict_in_one_memory(tmp_path) -> None:
|
||||
service, _, _ = _pair(tmp_path, newer_holder="husband")
|
||||
|
||||
assert service.conflicts("wife") == [], "the husband changing his own mind is not the wife's contradiction"
|
||||
|
||||
|
||||
def test_a_change_in_another_scenario_is_a_different_question(tmp_path) -> None:
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
database = Database(settings)
|
||||
database.init()
|
||||
service = MemoryService(settings=settings, database=database)
|
||||
service.remember(OLD, holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
service.remember(NEW, holder_id="wife", origin=MemoryOrigin.self_, scenario="office")
|
||||
|
||||
assert service.conflicts("wife") == [], (
|
||||
"the office thermostat and the home thermostat are two thermostats"
|
||||
)
|
||||
|
||||
|
||||
def test_a_supersede_already_chose_a_side_and_records_no_conflict(tmp_path) -> None:
|
||||
service, _, ids = _pair(tmp_path, scenario="")
|
||||
before = len(service.conflicts("wife"))
|
||||
|
||||
service.supersede(ids[0], text="恒温器调到 26 度", holder_id="wife")
|
||||
|
||||
assert len(service.conflicts("wife")) == before, (
|
||||
"a replacement is a decision, not a disagreement waiting to be settled"
|
||||
)
|
||||
|
||||
|
||||
def test_a_disagreement_about_words_rather_than_numbers_is_left_alone(tmp_path) -> None:
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
database = Database(settings)
|
||||
database.init()
|
||||
service = MemoryService(settings=settings, database=database)
|
||||
service.remember("钥匙在玄关抽屉里", holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
service.remember("钥匙在车里", holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
|
||||
assert service.conflicts("wife") == [], (
|
||||
"the numeric rule is the one that can be checked; a reworded sentence would fire on every paraphrase"
|
||||
)
|
||||
|
||||
|
||||
def test_an_open_conflict_makes_the_store_answer_with_the_newer_side(tmp_path) -> None:
|
||||
_, database, ids = _pair(tmp_path)
|
||||
retrieval = RetrievalService(settings=Settings(home=tmp_path, default_holder="wife"), database=database)
|
||||
|
||||
hits = retrieval.search("恒温器", filters=SearchFilters(as_holder="wife")).hits
|
||||
|
||||
assert [hit.atom.atom_id for hit in hits] == [ids[1]], (
|
||||
"both memories match the words; only the disagreement decides which one answers"
|
||||
)
|
||||
|
||||
|
||||
def test_settling_a_conflict_gives_both_sides_back(tmp_path) -> None:
|
||||
service, database, ids = _pair(tmp_path)
|
||||
retrieval = RetrievalService(settings=Settings(home=tmp_path, default_holder="wife"), database=database)
|
||||
clash = service.conflicts("wife")[0]
|
||||
|
||||
assert service.resolve_conflicts([clash.conflict_id], at=NOW) == 1
|
||||
assert service.conflicts("wife") == []
|
||||
|
||||
hits = retrieval.search("恒温器", filters=SearchFilters(as_holder="wife")).hits
|
||||
assert {hit.atom.atom_id for hit in hits} == set(ids), (
|
||||
"a settled disagreement stops the store from choosing; both memories are current again"
|
||||
)
|
||||
assert database.get_atoms(ids)[0].valid_to is None, "settling is not the same as superseding"
|
||||
|
||||
|
||||
def test_a_conflict_belongs_to_the_memory_it_was_found_in(tmp_path) -> None:
|
||||
service, _, ids = _pair(tmp_path)
|
||||
assert service.conflicts("wife"), "the disagreement exists before the forget"
|
||||
|
||||
forgotten = service.forget(atom_id=ids[1], reason="the reading was wrong")
|
||||
|
||||
assert forgotten.atoms_deleted == 1
|
||||
assert service.conflicts("wife") == [], (
|
||||
"one side is gone, so there is nothing left to settle and no ghost row to list"
|
||||
)
|
||||
@@ -242,3 +242,73 @@ def test_a_second_version_database_rebuilds_its_index_on_upgrade(tmp_path: Path)
|
||||
assert [atom_id for atom_id, _ in search_fts(upgraded, "钥匙", 5)] == ["key"]
|
||||
with upgraded.connect() as connection:
|
||||
assert int(connection.execute("PRAGMA user_version").fetchone()[0]) == SCHEMA_VERSION
|
||||
|
||||
|
||||
def test_a_fresh_database_carries_the_tier_columns_and_their_tables(tmp_path: Path) -> None:
|
||||
database = _database(tmp_path)
|
||||
columns = _atom_columns(database)
|
||||
for column in (
|
||||
"session_id",
|
||||
"importance",
|
||||
"recall_count",
|
||||
"last_recalled_at",
|
||||
"archived_at",
|
||||
):
|
||||
assert column in columns, column
|
||||
with database.connect() as connection:
|
||||
tables = {
|
||||
row[0]
|
||||
for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
|
||||
}
|
||||
assert {"consolidations", "conflicts"} <= tables, (
|
||||
"a lane's watermark and a store's disagreements are state, so they live in tables"
|
||||
)
|
||||
|
||||
|
||||
def test_the_current_view_hides_an_archived_memory_without_deleting_it(tmp_path: Path) -> None:
|
||||
database = _database(tmp_path)
|
||||
_seed_source(database)
|
||||
database.upsert_atoms([_atom("kept"), _atom("archived")])
|
||||
with database.connect() as connection:
|
||||
connection.execute(
|
||||
"UPDATE atoms SET archived_at = ? WHERE atom_id = 'archived'", (NOW.isoformat(),)
|
||||
)
|
||||
visible = connection.execute("SELECT atom_id FROM v_atoms_current ORDER BY atom_id").fetchall()
|
||||
assert [row[0] for row in visible] == ["kept"], "the archive is a tier, not a deletion"
|
||||
assert database.atom_count() == 2
|
||||
|
||||
|
||||
def test_a_ninth_version_database_gains_the_tiers_in_place(tmp_path: Path) -> None:
|
||||
"""A ninth-version store had no tiers at all; the upgrade must add them and rebuild the view."""
|
||||
database = _database(tmp_path)
|
||||
_seed_source(database)
|
||||
database.upsert_atoms([_atom("key")])
|
||||
with database.connect() as connection:
|
||||
# The downgrade removes what a ninth build never had, including the view and indexes
|
||||
# that reference those columns, so the reopened store really is a ninth-version one.
|
||||
connection.executescript(
|
||||
"""
|
||||
DROP VIEW IF EXISTS v_atoms_current;
|
||||
DROP INDEX IF EXISTS idx_atoms_session;
|
||||
DROP INDEX IF EXISTS idx_atoms_archived;
|
||||
ALTER TABLE atoms DROP COLUMN session_id;
|
||||
ALTER TABLE atoms DROP COLUMN importance;
|
||||
ALTER TABLE atoms DROP COLUMN recall_count;
|
||||
ALTER TABLE atoms DROP COLUMN last_recalled_at;
|
||||
ALTER TABLE atoms DROP COLUMN archived_at;
|
||||
DROP TABLE consolidations;
|
||||
DROP TABLE conflicts;
|
||||
PRAGMA user_version = 9;
|
||||
"""
|
||||
)
|
||||
|
||||
upgraded = _database(tmp_path)
|
||||
assert "archived_at" in _atom_columns(upgraded)
|
||||
assert [atom.atom_id for atom in upgraded.list_atoms()] == ["key"], "the upgrade kept the memories"
|
||||
with upgraded.connect() as connection:
|
||||
assert int(connection.execute("PRAGMA user_version").fetchone()[0]) == SCHEMA_VERSION
|
||||
connection.execute("UPDATE atoms SET archived_at = ? WHERE atom_id = 'key'", (NOW.isoformat(),))
|
||||
hidden = connection.execute("SELECT count(*) FROM v_atoms_current").fetchone()[0]
|
||||
assert hidden == 0, (
|
||||
"the old view body survived the upgrade, so an archived memory would still answer"
|
||||
)
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Memory tiers: a session reloads in order, a lane rolls up, and what nobody uses steps aside.
|
||||
|
||||
Nothing here deletes. Every tier move is reversible, because the store's guess about what
|
||||
mattered is a guess, and a wrong guess should cost a query, not a memory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.memory import MemoryService
|
||||
from ely_eye.retrieval import RetrievalService
|
||||
from ely_eye.schemas import MemoryOrigin, SearchFilters
|
||||
|
||||
NOW = datetime(2026, 5, 1, 12, 0, tzinfo=UTC)
|
||||
SESSION = "cooking-2026-10"
|
||||
|
||||
|
||||
def _service(tmp_path, holder: str = "wife") -> tuple[MemoryService, Database]:
|
||||
settings = Settings(home=tmp_path, default_holder=holder)
|
||||
database = Database(settings)
|
||||
database.init()
|
||||
return MemoryService(settings=settings, database=database), database
|
||||
|
||||
|
||||
def _remember(service: MemoryService, text: str, **kwargs) -> str:
|
||||
return service.remember(text, holder_id="wife", origin=MemoryOrigin.self_, **kwargs).atom_id
|
||||
|
||||
|
||||
def test_a_session_comes_back_in_the_order_it_was_recorded(tmp_path) -> None:
|
||||
service, _ = _service(tmp_path)
|
||||
first = _remember(service, "烤箱要除垢", session_id=SESSION)
|
||||
second = _remember(service, "面粉在储物架第二层", session_id=SESSION)
|
||||
|
||||
working = service.working_set("wife", SESSION)
|
||||
|
||||
assert [atom.atom_id for atom in working] == [first, second], (
|
||||
"a working set is the session's own timeline, not a ranking of it"
|
||||
)
|
||||
assert all(atom.session_id == SESSION for atom in working)
|
||||
|
||||
|
||||
def test_a_working_set_refuses_to_guess_which_session_is_working(tmp_path) -> None:
|
||||
service, _ = _service(tmp_path)
|
||||
_remember(service, "面粉在储物架第二层")
|
||||
|
||||
with pytest.raises(ValueError, match="a working set needs a session"):
|
||||
service.working_set("wife", "")
|
||||
|
||||
|
||||
def test_one_session_name_does_not_merge_two_holders(tmp_path) -> None:
|
||||
service, _ = _service(tmp_path)
|
||||
hers = _remember(service, "面粉在储物架第二层", session_id=SESSION)
|
||||
service.remember("车库里有备用钥匙", holder_id="husband", origin=MemoryOrigin.self_, session_id=SESSION)
|
||||
|
||||
assert [atom.atom_id for atom in service.working_set("wife", SESSION)] == [hers], (
|
||||
"a shared session name is not a shared memory"
|
||||
)
|
||||
|
||||
|
||||
def test_a_lane_rolls_up_once_and_keeps_rolling_from_where_it_stopped(tmp_path) -> None:
|
||||
service, database = _service(tmp_path)
|
||||
assert service.consolidate("wife", "home") is None, "an empty lane has nothing to summarise"
|
||||
|
||||
_remember(service, "烤箱要除垢", scenario="home")
|
||||
_remember(service, "面粉在储物架第二层", scenario="home")
|
||||
first = service.consolidate("wife", "home", at=NOW)
|
||||
assert first is not None and first.atom_count == 2
|
||||
rollup = database.get_atoms([first.rollup_atom_id])[0]
|
||||
assert "烤箱要除垢" in rollup.text and "面粉在储物架第二层" in rollup.text
|
||||
|
||||
assert service.consolidate("wife", "home", at=NOW) is None, (
|
||||
"the watermark is what stops the same memory from being counted twice"
|
||||
)
|
||||
|
||||
_remember(service, "恒温器调到 24 度", scenario="home")
|
||||
second = service.consolidate("wife", "home", at=NOW + timedelta(days=1))
|
||||
assert second is not None and second.atom_count == 3, (
|
||||
"the standing summary grows; it does not start over and forget what it already held"
|
||||
)
|
||||
grown = database.get_atoms([second.rollup_atom_id])[0]
|
||||
assert grown.atom_id == first.rollup_atom_id, "one lane holds one summary, replaced in place"
|
||||
assert "烤箱要除垢" in grown.text and "恒温器调到 24 度" in grown.text
|
||||
|
||||
|
||||
def test_another_scenario_is_a_different_lane(tmp_path) -> None:
|
||||
service, _ = _service(tmp_path)
|
||||
_remember(service, "烤箱要除垢", scenario="home")
|
||||
_remember(service, "周报周五交", scenario="work")
|
||||
|
||||
home = service.consolidate("wife", "home", at=NOW)
|
||||
assert home is not None and home.atom_count == 1
|
||||
work = service.consolidate("wife", "work", at=NOW)
|
||||
assert work is not None and work.atom_count == 1
|
||||
assert home.rollup_atom_id != work.rollup_atom_id
|
||||
|
||||
|
||||
def test_the_archive_rule_states_its_own_terms_and_can_be_read_before_it_acts(tmp_path) -> None:
|
||||
service, database = _service(tmp_path)
|
||||
forgotten = _remember(service, "前租户的 wifi 密码")
|
||||
valued = _remember(service, "妈妈的生日是 3 月 4 日", importance=0.9)
|
||||
future = NOW + timedelta(days=365)
|
||||
|
||||
preview = service.archive_stale(older_than_days=0, max_importance=0.5, at=future, dry_run=True)
|
||||
assert preview.dry_run and preview.atom_ids == [forgotten]
|
||||
assert "recorded before" in preview.rule, "the rule is stated, not hidden in a decay curve"
|
||||
assert service.archived() == []
|
||||
|
||||
moved = service.archive_stale(older_than_days=0, max_importance=0.5, at=future)
|
||||
assert moved.atom_ids == [forgotten] and not moved.dry_run
|
||||
assert [atom.atom_id for atom in service.archived("wife")] == [forgotten]
|
||||
assert database.get_atoms([forgotten])[0].archived_at is not None
|
||||
assert database.get_atoms([valued])[0].archived_at is None, (
|
||||
"a memory somebody marked important is not the archive's to move"
|
||||
)
|
||||
assert database.atom_count() == 2, "the archive steps a memory aside; it does not erase it"
|
||||
|
||||
|
||||
def test_an_archive_that_moves_nothing_says_so_rather_than_reporting_a_win(tmp_path) -> None:
|
||||
service, _ = _service(tmp_path)
|
||||
_remember(service, "烤箱要除垢")
|
||||
|
||||
moved = service.archive_stale(older_than_days=365, at=NOW)
|
||||
|
||||
assert moved.atom_ids == [], "a memory recorded yesterday is not older than a year"
|
||||
|
||||
|
||||
def test_restoring_brings_the_memory_back_to_the_current_tier(tmp_path) -> None:
|
||||
service, database = _service(tmp_path)
|
||||
atom_id = _remember(service, "前租户的 wifi 密码")
|
||||
service.archive_stale(older_than_days=0, at=NOW + timedelta(days=365))
|
||||
|
||||
assert service.restore([atom_id]) == 1
|
||||
assert service.archived() == []
|
||||
assert database.get_atoms([atom_id])[0].archived_at is None
|
||||
|
||||
with pytest.raises(ValueError, match="restore needs atom ids"):
|
||||
service.restore([])
|
||||
|
||||
|
||||
def test_a_search_counts_the_memory_it_handed_back(tmp_path) -> None:
|
||||
service, database = _service(tmp_path)
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
retrieval = RetrievalService(settings=settings, database=database)
|
||||
atom_id = _remember(service, "我们的钥匙放在玄关抽屉里")
|
||||
|
||||
hits = retrieval.search("钥匙", filters=SearchFilters(as_holder="wife")).hits
|
||||
assert any(hit.atom.atom_id == atom_id for hit in hits)
|
||||
|
||||
used = database.get_atoms([atom_id])[0]
|
||||
assert used.recall_count == 1, "the store records that this memory was used, which is the only use estimate it has"
|
||||
assert used.last_recalled_at is not None
|
||||
|
||||
retrieval.search("钥匙", filters=SearchFilters(as_holder="wife"))
|
||||
assert database.get_atoms([atom_id])[0].recall_count == 2
|
||||
|
||||
|
||||
def test_a_memory_a_search_already_returned_is_not_stale(tmp_path) -> None:
|
||||
service, database = _service(tmp_path)
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
retrieval = RetrievalService(settings=settings, database=database)
|
||||
atom_id = _remember(service, "我们的钥匙放在玄关抽屉里")
|
||||
assert retrieval.search("钥匙", filters=SearchFilters(as_holder="wife")).hits
|
||||
|
||||
moved = service.archive_stale(older_than_days=0, at=NOW + timedelta(days=365))
|
||||
|
||||
assert moved.atom_ids == [], (
|
||||
"a memory that keeps coming back was used; the archive takes what nobody ever asked for"
|
||||
)
|
||||
assert database.get_atoms([atom_id])[0].archived_at is None
|
||||
|
||||
|
||||
def test_an_archived_memory_stops_answering_without_leaving_the_store(tmp_path) -> None:
|
||||
service, database = _service(tmp_path)
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
retrieval = RetrievalService(settings=settings, database=database)
|
||||
atom_id = _remember(service, "我们的钥匙放在玄关抽屉里")
|
||||
|
||||
service.archive_stale(older_than_days=0, at=NOW + timedelta(days=365))
|
||||
|
||||
assert retrieval.search("钥匙", filters=SearchFilters(as_holder="wife")).hits == []
|
||||
assert database.get_atoms([atom_id])[0].archived_at is not None, (
|
||||
"the row is still there; only the current tier stopped carrying it"
|
||||
)
|
||||
|
||||
|
||||
def test_importance_is_a_stated_value_and_not_a_free_number(tmp_path) -> None:
|
||||
from pydantic import ValidationError
|
||||
|
||||
from ely_eye.schemas import EvidenceAtom, Modality, TrustScores
|
||||
|
||||
with pytest.raises(ValidationError, match="less than or equal to 1"):
|
||||
EvidenceAtom(
|
||||
atom_id="mem-1:text",
|
||||
modality=Modality.text,
|
||||
source_id="mem-1",
|
||||
source="memory://wife/home",
|
||||
text="恒温器调到 24 度",
|
||||
trust=TrustScores(parser=1.0),
|
||||
holder_id="wife",
|
||||
importance=1.5,
|
||||
)
|
||||
@@ -40,8 +40,10 @@ HTTP_SURFACE = {
|
||||
"/api/ingest/path",
|
||||
"/api/ingest/upload",
|
||||
"/api/memory",
|
||||
"/api/memory/conflicts",
|
||||
"/api/memory/forget",
|
||||
"/api/memory/supersede",
|
||||
"/api/memory/working",
|
||||
"/api/mounts",
|
||||
"/api/proof-suite",
|
||||
"/api/proof-suite/latest",
|
||||
|
||||
Reference in new issue
Block a user