ZacharyZhang-NY/ely-eye
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Answer from the moment a question asks about

Browse Source 
The key test in Gaps.md §6 asks what was true last week. It could not be
answered: search_atoms joined v_atoms_current, so a superseded memory is
unreachable no matter what since/until says — those two fields bound when a
memory was true, not which moment the store answers from.

SearchFilters.as_of moves that moment. Without it the words leg repeats the
view's currency predicate verbatim; with it a memory answers when
valid_from <= as_of < valid_to, its event time has arrived, and it is inside
its retention. archived_at is ignored on purpose: archival is a storage tier,
not a claim about the world, so an archived memory still answers for the time
before it was archived. The dense leg reads the same window through
vector_index._allowed, so the two legs cannot disagree about what existed.

scripts/key_test.sh runs the seven §6 scenarios end to end against one live
store — HTTP, the Go MCP binary over stdio, and the CLI mounting cartridges.
Six pass. The seventh, "a cartridge saves compute", is reported NOT MEASURED
rather than passed: it claims time-to-first-token and a cache-hit rate, no
generator has ever run here, and there is no compile or cache route to observe
even the second half.

Gates: pytest -q 406 passed; ruff clean; filesize OK; pyright 0 on the seven
touched files; mypy cold cache Success on the same seven; key test exit 0.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-09 18:25:30 -04:00
1 parent 89c880fd36
commit eab6bd7c8c
7 files changed
+644 -22
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/src/ely_eye/cli.py
+18 -4
View File
@@ -88,14 +88,25 @@ def search(
query: str,
top_k: int = typer.Option(0, "--top-k", help="hits to return; 0 uses the configured default"),
holder: str = typer.Option("", "--holder", help="only this holder's memories may answer"),
as_holder: str = typer.Option("", "--as-holder", help="see the store through this holder's eyes, the way their agent would"),
as_holder: str = typer.Option(
"",
"--as-holder",
help="see the store through this holder's eyes, the way their agent would",
),
scenario: str = typer.Option("", "--scenario"),
source_id: str = typer.Option("", "--source-id"),
since: str = typer.Option("", "--since", help="event time at or after, ISO with offset"),
until: str = typer.Option("", "--until"),
min_trust: float = typer.Option(-1.0, "--min-trust", help="drop memories whose trust falls below this"),
as_of: str = typer.Option(
"", "--as-of", help="answer as if it were this moment; memories retired since still answer"
),
min_trust: float = typer.Option(
-1.0, "--min-trust", help="drop memories whose trust falls below this"
),
cursor: str = typer.Option("", "--cursor", help="next_cursor from the previous page"),
image: str = typer.Option("", "--image", help="a picture to search with, instead of only words"),
image: str = typer.Option(
"", "--image", help="a picture to search with, instead of only words"
),
) -> None:
"""Rank the memories that may answer, and say so when none may."""
settings = get_settings()
@@ -109,6 +120,7 @@ def search(
source_id=source_id or None,
since=moment(since, "--since"),
until=moment(until, "--until"),
as_of=moment(as_of, "--as-of"),
min_trust=None if min_trust < 0 else min_trust,
),
cursor=cursor or None,
@@ -275,7 +287,9 @@ def runtime_profiles() -> None:
@app.command("research-modules")
def research_modules(query: str = "Ely-Eye V-NSA DVKV x-modal KV speculative prefetch evidence graph") -> None:
def research_modules(
query: str = "Ely-Eye V-NSA DVKV x-modal KV speculative prefetch evidence graph",
) -> None:
settings = get_settings()
db = Database(settings)
from .compiler import ContextCompiler
backend/src/ely_eye/retrieval.py
+1
View File
@@ -116,6 +116,7 @@ class RetrievalService:
modality=criteria.get("modality"),
since=_text(criteria.get("since")),
until=_text(criteria.get("until")),
as_of=_text(criteria.get("as_of")),
cartridge_ids=criteria.get("cartridge_ids"),
as_holder=criteria.get("as_holder"),
)
backend/src/ely_eye/schemas.py
+18 -1
View File
@@ -125,7 +125,16 @@ class EvidenceAtom(BaseModel):
raise ValueError(f"{self.atom_id}: an atom cannot supersede itself")
return self
@field_validator("ingested_at", "event_time", "event_time_end", "valid_from", "valid_to", "retention_until", "last_recalled_at", "archived_at")
@field_validator(
"ingested_at",
"event_time",
"event_time_end",
"valid_from",
"valid_to",
"retention_until",
"last_recalled_at",
"archived_at",
)
@classmethod
def _times_are_aware(cls, value: datetime | None) -> datetime | None:
if value is not None and value.tzinfo is None:
@@ -172,6 +181,13 @@ class SearchFilters(BaseModel):
A memory with no event time cannot satisfy since/until, so it drops out when a time bound is
asked for rather than passing as unknown. `holder` restricts the answer to one holder's
memories; `as_holder` is the identity asking, and applies the sharing rule instead.
`since`/`until` bound *when a memory was true*; `as_of` moves the moment the store answers
from, which is a different question. Asked as of last Tuesday, a memory that was true then
may answer even if it has been retired since, and a memory whose event time falls after that
moment had not happened yet, so it may not. Archival is a storage tier rather than a claim
about the world, so an archived memory still answers a question about the time before it was
archived. Without `as_of` only memories open right now may answer.
"""
holder: str | None = None
@@ -180,6 +196,7 @@ class SearchFilters(BaseModel):
source_id: str | None = None
modality: Modality | None = None
cartridge_ids: list[str] | None = None
as_of: datetime | None = None
since: datetime | None = None
until: datetime | None = None
min_trust: float | None = Field(default=None, ge=0.0, le=1.0)
backend/src/ely_eye/store_search.py
+29 -2
View File
@@ -36,29 +36,50 @@ def search_atoms(
modality: str | None = None,
since: datetime | None = None,
until: datetime | None = None,
as_of: datetime | None = None,
min_trust: float | None = None,
cartridge_ids: list[str] | None = None,
as_holder: str | None = None,
) -> FtsPage:
"""BM25 over the FTS index joined to memories that are open right now and readable by the asker.
"""BM25 over the FTS index joined to memories that answer for the moment being asked about.
Every filter is a `? IS NULL OR` predicate in one literal statement: the ranking never
depends on string-built SQL, and the planner sees one shape. A memory with no event time
cannot answer a time filter, so it drops out when one is asked for.
With no `as_of` the join repeats `v_atoms_current` in place of the view — same rule, asked
through the atom_id index instead of a scan. With one, the question is what was true then,
so a memory retired since still answers and `archived_at` (a storage tier, not a claim about
the world) is ignored.
`as_holder` applies the sharing rule in three states: a holder id sees its own memories, the
ones naming it in visible_to, the ones marked for everyone, and the ones nobody owns; the
empty string is a caller who is nobody, so only the last two; None runs over the whole
library, which is what an operator or a process that owns the store reads with. A memory with
no holder cannot be private to a holder that does not exist."""
as_of_iso = None if as_of is None else as_of.isoformat()
with database.connect() as connection:
expression = fts_expression(rare_terms(connection, fts_terms(query)))
rows = connection.execute(
"""
SELECT f.atom_id, bm25(atom_fts) AS score
FROM atom_fts AS f
JOIN v_atoms_current AS a ON a.atom_id = f.atom_id
JOIN atoms AS a ON a.atom_id = f.atom_id
WHERE atom_fts MATCH ?
AND (
(? IS NULL
AND a.valid_to IS NULL
AND a.archived_at IS NULL
AND (a.valid_from IS NULL
OR a.valid_from <= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
AND (a.retention_until IS NULL
OR a.retention_until >= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now')))
OR (? IS NOT NULL
AND (a.valid_from IS NULL OR a.valid_from <= ?)
AND (a.valid_to IS NULL OR a.valid_to > ?)
AND (a.event_time IS NULL OR a.event_time <= ?)
AND (a.retention_until IS NULL OR a.retention_until > ?))
)
AND (? IS NULL OR a.holder_id = ?)
AND (? IS NULL OR a.scenario = ?)
AND (? IS NULL OR a.source_id = ?)
@@ -91,6 +112,12 @@ def search_atoms(
""",
(
expression,
as_of_iso,
as_of_iso,
as_of_iso,
as_of_iso,
as_of_iso,
as_of_iso,
holder,
holder,
scenario,
backend/src/ely_eye/vector_index.py
+67 -15
View File
@@ -61,7 +61,10 @@ class VectorIndex:
"""One float32 vector per current atom: the writer keeps it, the reader asks it questions."""
def __init__(
self, settings: Settings | None = None, database: Database | None = None, embeddings: EmbeddingService | None = None
self,
settings: Settings | None = None,
database: Database | None = None,
embeddings: EmbeddingService | None = None,
) -> None:
self.settings = settings or get_settings()
self.db = database or Database(self.settings)
@@ -99,7 +102,12 @@ class VectorIndex:
self.ensure(connection, dim)
stale = self._stale(connection)
if not stale:
return {"encoded": 0, "dim": dim, "model": self.settings.embedding_model_id, "seconds": 0.0}
return {
"encoded": 0,
"dim": dim,
"model": self.settings.embedding_model_id,
"seconds": 0.0,
}
vectors = self.embeddings.encode_atoms([atom for _, atom in stale])
self._write(connection, stale, vectors)
return {
@@ -124,10 +132,18 @@ class VectorIndex:
self.ensure(connection, dim)
current = self._current(connection, wanted)
kept = {atom_id for atom_id, _ in current}
dropped = self._delete(connection, [atom_id for atom_id in wanted if atom_id not in kept])
dropped = self._delete(
connection, [atom_id for atom_id in wanted if atom_id not in kept]
)
if current:
self._write(connection, current, self.embeddings.encode_atoms([atom for _, atom in current]))
return {"encoded": len(current), "dropped": dropped, "seconds": round(time.monotonic() - started, 3)}
self._write(
connection, current, self.embeddings.encode_atoms([atom for _, atom in current])
)
return {
"encoded": len(current),
"dropped": dropped,
"seconds": round(time.monotonic() - started, 3),
}
def search(
self,
@@ -140,6 +156,7 @@ class VectorIndex:
modality: str | None = None,
since: str | None = None,
until: str | None = None,
as_of: str | None = None,
cartridge_ids: list[str] | None = None,
as_holder: str | None = None,
) -> dict[str, float]:
@@ -156,7 +173,9 @@ class VectorIndex:
"""
if limit <= 0:
raise ValueError(f"dense search needs a positive limit, got {limit}")
scope = None if cartridge_ids is None else sorted(atoms_of_cartridges(self.db, cartridge_ids))
scope = (
None if cartridge_ids is None else sorted(atoms_of_cartridges(self.db, cartridge_ids))
)
if scope == []:
return {}
narrow = to_json(scope) if scope is not None and len(scope) <= SCOPE_NARROW_MAX else None
@@ -178,10 +197,13 @@ class VectorIndex:
modality=modality,
since=since,
until=until,
as_of=as_of,
cartridge_ids=cartridge_ids,
as_holder=as_holder,
)
hits = {atom_id: 1.0 - distance for atom_id, distance in ranked if atom_id in allowed}
hits = {
atom_id: 1.0 - distance for atom_id, distance in ranked if atom_id in allowed
}
# A filter that keeps little of the ask means more answers sit further out; one wider round buys them, bounded by the ceiling.
if len(hits) >= limit or len(ranked) < asked or asked >= CANDIDATE_CEILING:
break
@@ -216,6 +238,7 @@ class VectorIndex:
modality: str | None,
since: str | None,
until: str | None,
as_of: str | None,
cartridge_ids: list[str] | None,
as_holder: str | None,
) -> set[str]:
@@ -223,7 +246,10 @@ class VectorIndex:
The currency rules repeat `v_atoms_current` here on purpose: the view answers this
question with a full scan (77ms over 153,857 atoms) while the atoms table answers it
through the atom_id index in 0.2ms, and a search cannot afford the scan. The scope is
through the atom_id index in 0.2ms, and a search cannot afford the scan. An `as_of`
replaces that rule with the window that was true then, retired memories included, and
ignores `archived_at` because archival is a storage tier rather than a claim about the
world. The scope is
checked here rather than in the engine so the rule lives in one place; the engine's id
list is only a narrowing. Visibility is here too, and it reads the same three states the
words leg reads: a holder id, the empty string for a caller who is nobody (unowned rows
@@ -233,10 +259,20 @@ class VectorIndex:
"""
SELECT atom_id FROM atoms
WHERE atom_id IN (SELECT value FROM json_each(?))
AND archived_at IS NULL
AND (valid_to IS NULL OR valid_to > strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
AND (valid_from IS NULL OR valid_from <= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
AND (retention_until IS NULL OR retention_until >= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
AND (
(? IS NULL
AND archived_at IS NULL
AND (valid_to IS NULL OR valid_to > strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
AND (valid_from IS NULL
OR valid_from <= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now'))
AND (retention_until IS NULL
OR retention_until >= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now')))
OR (? IS NOT NULL
AND (valid_from IS NULL OR valid_from <= ?)
AND (valid_to IS NULL OR valid_to > ?)
AND (event_time IS NULL OR event_time <= ?)
AND (retention_until IS NULL OR retention_until > ?))
)
AND (? IS NULL OR holder_id = ?)
AND (? IS NULL OR scenario = ?)
AND (? IS NULL OR source_id = ?)
@@ -259,6 +295,12 @@ class VectorIndex:
""",
(
to_json(atom_ids),
as_of,
as_of,
as_of,
as_of,
as_of,
as_of,
holder,
holder,
scenario,
@@ -290,7 +332,14 @@ class VectorIndex:
with self.db.connect() as connection:
meta = self._meta(connection)
if not meta:
return {"built": False, "format": None, "dim": 0, "vectors": 0, "expected": 0, "pending": 0}
return {
"built": False,
"format": None,
"dim": 0,
"vectors": 0,
"expected": 0,
"pending": 0,
}
rows = connection.execute("SELECT count(*) AS n FROM vec_atoms").fetchone()["n"]
expected = _expected(connection)
pending = len(self._stale(connection))
@@ -355,12 +404,15 @@ class VectorIndex:
raise ValueError(f"sync encoded {len(vectors)} vectors for {len(items)} atoms")
matrix = np.asarray(vectors, dtype=np.float32)
if matrix.ndim != 2 or matrix.shape[1] != self._dim(connection):
raise ValueError(f"sync produced a {matrix.shape} matrix, expected one row per atom at the stored width")
raise ValueError(
f"sync produced a {matrix.shape} matrix, expected one row per atom at the stored width"
)
ids = [atom_id for atom_id, _ in items]
for start in range(0, len(ids), DELETE_CHUNK):
stop = min(start + DELETE_CHUNK, len(ids))
connection.execute(
"DELETE FROM vec_atoms WHERE atom_id IN (SELECT value FROM json_each(?))", (to_json(ids[start:stop]),)
"DELETE FROM vec_atoms WHERE atom_id IN (SELECT value FROM json_each(?))",
(to_json(ids[start:stop]),),
)
for start in range(0, len(items), BATCH):
stop = min(start + BATCH, len(items))