Give memories a working tier, a long-term tier, and a record of what disagrees
A memory used to have one state: present. A session could not be reloaded
without searching, nothing separated what a person said today from what the
store ingested a year ago, and two statements that disagree about a number
were both handed to the reader as if the store believed both.
Schema v10. Atoms gain session_id, importance, recall_count, last_recalled_at
and archived_at; two tables arrive beside them — consolidations (one watermark
per holder+scenario lane) and conflicts (a pair, the term they disagree on,
both values, and whether anyone has settled it). v_atoms_current now also
hides archived rows, and apply() drops the view before rebuilding it because
CREATE VIEW IF NOT EXISTS would keep the old body and ignore the archive.
Working memory is a read, not a store: `working <holder> <session>` returns
what that session recorded, in order, with no search involved. Consolidation
rolls a lane's new memories into one standing summary and moves the watermark
to the newest memory actually rolled up — not to the clock of the pass, which
made a second pass count the same memories twice. The archive rule is stated
in the command that runs it (recorded before N days, never returned by a
search, importance at or under a floor) rather than hidden in a decay formula
nobody measured; --dry-run names the victims first.
A conflict is recorded when a new memory repeats a label with a different
number in the same holder's same scenario. Retrieval drops the older side
while the conflict is open and both sides answer again once it is settled:
settling is not superseding, and a word-only disagreement is left alone
because a reworded sentence would fire on every paraphrase. Numbers are keyed
to both the memory's subject and the nearest preceding label, measured: the
two thermostat sentences share only their subject, so a nearest-word-only rule
found nothing.
Search now counts the memories it handed back, which is the only use estimate
the store has, and the archive rule reads it. The dense index applies the same
currency rules as the view, so an archived memory cannot stay findable through
one leg while the other hides it.
Two reads over HTTP (working, conflicts) and two tools over MCP
(working_set, memory_conflicts); curation stays with the operator because an
agent should not decide what its own past is worth. remember and supersede
carry a session id; importance does not cross MCP, because recall_count is
observed and importance is stated.
Gates: 366 passed; ruff clean; mypy 24 errors in 9 files, the recorded 9cb46c1
baseline, net 0; pyright 0 on the touched files; go build/vet/test and gofmt
clean; mcp_write_smoke PASS.
This commit is contained in:
1 parent
4c6af87fe0
commit
dc9cfb5a30
30 files changed
+1814
-229
No files matched your search
@@ -9,15 +9,18 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import trust
|
||||
from . import lifecycle_store, trust
|
||||
from .cache_fabric import CacheFabric
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .schemas import (
|
||||
ArchiveReport,
|
||||
ConflictRecord,
|
||||
ConsolidationRecord,
|
||||
EvidenceAtom,
|
||||
ForgetReport,
|
||||
MemoryOrigin,
|
||||
@@ -32,9 +35,12 @@ from .storage import token_equivalent
|
||||
from .vector_index import VectorIndex
|
||||
from .visual_index import VisualIndex
|
||||
|
||||
# Trust scores describe extraction fidelity, not truthfulness: a remembered sentence is stored verbatim, so origin carries the epistemic weight.
|
||||
# A remembered sentence is stored verbatim, so trust scores carry no weight here; origin does.
|
||||
MEMORY_TRUST = TrustScores(parser=1.0, ocr=0.0, model=0.0)
|
||||
|
||||
# The rollup atom's id suffix: one standing summary per holder and scenario, replaced in place.
|
||||
ROLLUP_SUFFIX = ":rollup"
|
||||
|
||||
|
||||
class MemoryService:
|
||||
def __init__(
|
||||
@@ -63,6 +69,8 @@ class MemoryService:
|
||||
supersedes: list[str] | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
retention_until: datetime | None = None,
|
||||
session_id: str = "",
|
||||
importance: float | None = None,
|
||||
) -> EvidenceAtom:
|
||||
"""Store one statement as a memory owned by a holder.
|
||||
|
||||
@@ -116,12 +124,44 @@ class MemoryService:
|
||||
valid_from=valid_from,
|
||||
visibility=visibility or Visibility(),
|
||||
retention_until=retention_until,
|
||||
session_id=session_id,
|
||||
importance=importance,
|
||||
)
|
||||
trust.stamp([atom], origin=origin)
|
||||
self.db.upsert_atoms([atom])
|
||||
self.vectors.sync_atoms([atom.atom_id])
|
||||
self._note_conflicts(atom)
|
||||
return atom
|
||||
|
||||
def _note_conflicts(self, atom: EvidenceAtom) -> None:
|
||||
"""A number that moves inside one holder's scenario is a disagreement, not a second fact.
|
||||
|
||||
Only a memory that replaces nothing can start a conflict: a supersede has already
|
||||
picked a side, and recording it as unresolved would contradict that.
|
||||
"""
|
||||
if atom.supersedes:
|
||||
return
|
||||
stated = trust.numeric_values(atom.text)
|
||||
if not stated:
|
||||
return
|
||||
for earlier in lifecycle_store.lane_atoms(self.db, atom.holder_id, atom.scenario):
|
||||
if earlier.atom_id == atom.atom_id:
|
||||
continue
|
||||
term = trust.conflicting_number(trust.numeric_values(earlier.text), stated)
|
||||
if term is None:
|
||||
continue
|
||||
lifecycle_store.record_conflict(
|
||||
self.db,
|
||||
older_atom_id=earlier.atom_id,
|
||||
newer_atom_id=atom.atom_id,
|
||||
holder_id=atom.holder_id,
|
||||
scenario=atom.scenario,
|
||||
terms=[term],
|
||||
older_value=trust.numeric_values(earlier.text)[term],
|
||||
newer_value=stated[term],
|
||||
detected_at=utc_now(),
|
||||
)
|
||||
|
||||
def supersede(
|
||||
self,
|
||||
atom_id: str,
|
||||
@@ -133,6 +173,8 @@ class MemoryService:
|
||||
event_time: datetime | None = None,
|
||||
event_time_precision: TimePrecision = TimePrecision.unknown,
|
||||
event_time_source: str = "",
|
||||
session_id: str = "",
|
||||
importance: float | None = None,
|
||||
) -> EvidenceAtom:
|
||||
"""Close an old memory and record the new one as its replacement.
|
||||
|
||||
@@ -159,6 +201,8 @@ class MemoryService:
|
||||
event_time_source=event_time_source,
|
||||
supersedes=[atom_id],
|
||||
valid_from=moment,
|
||||
session_id=session_id,
|
||||
importance=importance,
|
||||
)
|
||||
self.db.close_atoms([atom_id], moment)
|
||||
self.vectors.sync_atoms([atom_id])
|
||||
@@ -206,6 +250,144 @@ class MemoryService:
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
def working_set(self, holder_id: str, session_id: str) -> list[EvidenceAtom]:
|
||||
"""What this session recorded, in the order it recorded it: the short-term tier.
|
||||
|
||||
No search is involved. A session asking for its own context gets exactly what it put
|
||||
down, which is the whole point of a working set rather than a wider top-k.
|
||||
"""
|
||||
if not session_id.strip():
|
||||
raise ValueError("a working set needs a session; without one every memory is working memory")
|
||||
return lifecycle_store.session_atoms(self.db, holder_id, session_id)
|
||||
|
||||
def consolidate(
|
||||
self, holder_id: str, scenario: str = "", *, at: datetime | None = None
|
||||
) -> ConsolidationRecord | None:
|
||||
"""Roll a lane's new memories into one standing summary and move the watermark.
|
||||
|
||||
The summary replaces its own previous version under a fixed id, so a lane holds one
|
||||
rollup instead of a stack of them, and the ledger's watermark is what keeps a memory out
|
||||
of the next pass. Nothing is deleted: the individual memories stay current and findable,
|
||||
because a summary that erased the detail could not be questioned about where it came from.
|
||||
"""
|
||||
moment = at or utc_now()
|
||||
ledger = lifecycle_store.read_ledger(self.db, holder_id, scenario)
|
||||
batch = [
|
||||
atom
|
||||
for atom in lifecycle_store.lane_atoms(
|
||||
self.db, holder_id, scenario, after=ledger.watermark_at if ledger else None
|
||||
)
|
||||
if not atom.atom_id.endswith(ROLLUP_SUFFIX)
|
||||
]
|
||||
if not batch:
|
||||
return None
|
||||
|
||||
lines: list[str] = []
|
||||
if ledger:
|
||||
previous = self.db.get_atoms([ledger.rollup_atom_id])
|
||||
if previous:
|
||||
lines.append(previous[0].text)
|
||||
lines.extend(atom.text for atom in batch)
|
||||
body = "\n".join(lines)
|
||||
|
||||
digest = hashlib.sha256(f"{holder_id}\n{scenario}".encode()).hexdigest()
|
||||
source_id = f"rollup-{digest[:16]}"
|
||||
atom_id = f"{source_id}{ROLLUP_SUFFIX}"
|
||||
self.db.upsert_sources(
|
||||
[
|
||||
SourceRecord(
|
||||
source_id=source_id,
|
||||
path=f"memory://{holder_id}/{scenario or 'general'}#rollup",
|
||||
kind="memory",
|
||||
mime="text/plain",
|
||||
sha256=hashlib.sha256(body.encode("utf-8")).hexdigest(),
|
||||
size_bytes=len(body.encode("utf-8")),
|
||||
created_at=moment,
|
||||
metadata={"holder_id": holder_id, "rollup": True},
|
||||
)
|
||||
]
|
||||
)
|
||||
rollup = EvidenceAtom(
|
||||
atom_id=atom_id,
|
||||
modality=Modality.text,
|
||||
source_id=source_id,
|
||||
source=f"memory://{holder_id}/{scenario or 'general'}#rollup",
|
||||
ingested_at=moment,
|
||||
text=body,
|
||||
trust=MEMORY_TRUST,
|
||||
token_equivalent=token_equivalent(body),
|
||||
holder_id=holder_id,
|
||||
origin=MemoryOrigin.self_,
|
||||
scenario=scenario,
|
||||
)
|
||||
self.db.upsert_atoms([rollup])
|
||||
self.vectors.sync_atoms([atom_id])
|
||||
# The watermark is the newest memory actually rolled up, not the clock of the pass: a
|
||||
# consolidation run with a clock behind the memories would otherwise count them twice.
|
||||
record = ConsolidationRecord(
|
||||
holder_id=holder_id,
|
||||
scenario=scenario,
|
||||
watermark_at=max(atom.ingested_at for atom in batch),
|
||||
rollup_atom_id=atom_id,
|
||||
atom_count=(ledger.atom_count if ledger else 0) + len(batch),
|
||||
token_equivalent=token_equivalent(body),
|
||||
consolidated_at=moment,
|
||||
)
|
||||
lifecycle_store.write_ledger(self.db, record)
|
||||
return record
|
||||
|
||||
def archive_stale(
|
||||
self,
|
||||
*,
|
||||
older_than_days: int,
|
||||
max_importance: float | None = None,
|
||||
at: datetime | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> ArchiveReport:
|
||||
"""Move memories that were never used and never valued out of the current tier.
|
||||
|
||||
The rule is stated instead of hidden in a decay formula: a memory goes when it was
|
||||
recorded before the cutoff, no search ever returned it, and nobody marked it important.
|
||||
Archived rows can be brought back; deleting them would make a wrong judgement
|
||||
unrecoverable, and the store's own estimate of what mattered is exactly the kind of thing
|
||||
that turns out to be wrong.
|
||||
"""
|
||||
if older_than_days < 0:
|
||||
raise ValueError(f"older_than_days must not be negative, got {older_than_days}")
|
||||
moment = at or utc_now()
|
||||
cutoff = moment - timedelta(days=older_than_days)
|
||||
candidates = lifecycle_store.stale_candidates(self.db, before=cutoff, max_importance=max_importance)
|
||||
moved = 0 if dry_run else lifecycle_store.archive_atoms(self.db, candidates, moment)
|
||||
if not dry_run and moved != len(candidates):
|
||||
raise ValueError(f"{len(candidates)} memories were chosen for the archive but {moved} were moved")
|
||||
floor = "any importance" if max_importance is None else f"importance at or under {max_importance}"
|
||||
return ArchiveReport(
|
||||
atom_ids=candidates,
|
||||
archived_at=moment,
|
||||
rule=f"recorded before {cutoff.isoformat()}, never returned by a search, {floor}",
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
def archived(self, holder_id: str | None = None) -> list[EvidenceAtom]:
|
||||
"""The long-term tier: what the current view hides, listed rather than guessed at."""
|
||||
return lifecycle_store.archived_atoms(self.db, holder_id)
|
||||
|
||||
def restore(self, atom_ids: list[str]) -> int:
|
||||
"""Bring archived memories back: the archive is a tier, not a verdict."""
|
||||
if not atom_ids:
|
||||
raise ValueError("restore needs atom ids; nothing was named")
|
||||
return lifecycle_store.restore_atoms(self.db, atom_ids)
|
||||
|
||||
def conflicts(self, holder_id: str | None = None) -> list[ConflictRecord]:
|
||||
"""Every disagreement the store noticed and nobody has settled."""
|
||||
return lifecycle_store.open_conflicts(self.db, holder_id)
|
||||
|
||||
def resolve_conflicts(self, conflict_ids: list[int], *, at: datetime | None = None) -> int:
|
||||
"""Settle disagreements so retrieval stops dropping a side on the store's own reading."""
|
||||
if not conflict_ids:
|
||||
raise ValueError("resolving needs conflict ids; nothing was named")
|
||||
return lifecycle_store.resolve_conflicts(self.db, conflict_ids, at or utc_now())
|
||||
|
||||
def _drop_derived_vectors(self, atom_ids: list[str]) -> tuple[int, bool]:
|
||||
"""A forgotten memory must stop answering from the derived indexes too.
|
||||
|
||||
|
||||
Reference in new issue
Block a user