Keep the vectors beside the atoms instead of rebuilding them per query

The dense leg used to be a pair of files next to the database: a full .npy
re-encoded from every atom, re-normalized on every query, and stale the moment
anything was ingested. It is now a vec0 table inside the same SQLite file,
written in the same transaction as the atom it describes, so a memory is
queryable the instant it is remembered.

- vector_index.py maintains the table; the writer inserts, the reader never
  rebuilds. A repair scan (ely-eye build-dense-index) catches what a crash hid,
  and it is not on the search path.
- Ranking stays in the engine; every boundary (closed, retired, not yet true,
  another holder's private memory, the mounted cartridge) is checked once against
  the atoms table afterwards. A visibility subquery inside the KNN cost 48.9s on
  153,857 vectors; the same check over 200 candidates costs 0.2ms. When the
  boundaries reject enough candidates the ask widens once, never twice.
- The Go mirror and the web panel carry the new status fields.
- bench/dense_scale.py and bench/quant_probe.py measure the leg at corpus size;
  bench/scale_check.py now works on a copy so the frozen baseline stays frozen.

Measured on the frozen 165,973-atom corpus. bench/reports/DENSE.md, QUANT.md and
SCALE.md carry the numbers and the machine they were taken on.

Gates: 285 pytest, ruff clean, pyright 0 on the touched files, go build/vet/test
and gofmt clean, frontend build and eslint clean. mypy is 25 findings against the
parent's 24: the new one is "Skipping analyzing sqlite_vec: missing library stubs",
the same class as the two existing qwen_vl_utils lines, and the repo keeps no
type: ignore anywhere.

backend/tests/conftest.py

@@ -2,11 +2,13 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 
+import numpy as np
 import pytest
 
 ROPE: dict[str, Any] = {
@@ -51,3 +53,42 @@ def write_model_fixture() -> Callable[..., Path]:
 @pytest.fixture
 def model_dir(tmp_path: Path) -> Path:
     return write_model(tmp_path / "model")
+
+
+class HashEmbeddings:
+    """A stand-in for the embedding model: the same words give the same vector, no download.
+
+    Different words land in different directions, which is all a retrieval test needs; the
+    numbers mean nothing to a human, so no test should assert on a score's value.
+    """
+
+    def __init__(self, dim: int = 16) -> None:
+        self.dim = dim
+
+    def enabled(self) -> bool:
+        return True
+
+    def encode_texts(self, texts: list[str]) -> np.ndarray:
+        if not texts:
+            raise ValueError("encode_texts was asked to encode nothing")
+        rows = [
+            np.frombuffer(hashlib.sha256(text.encode("utf-8")).digest()[: self.dim], dtype=np.int8).astype(np.float32)
+            for text in texts
+        ]
+        matrix = np.stack(rows)
+        norms = np.linalg.norm(matrix, axis=1, keepdims=True)
+        return matrix / np.where(norms == 0, 1.0, norms)
+
+    def encode_atoms(self, atoms: list[Any]) -> np.ndarray:
+        return self.encode_texts([atom.text or "" for atom in atoms])
+
+    def embed_query(self, query: str) -> np.ndarray:
+        return self.encode_texts([query])[0]
+
+    def probe_dim(self) -> int:
+        return self.dim
+
+
+@pytest.fixture
+def fake_embeddings() -> Callable[[], HashEmbeddings]:
+    return HashEmbeddings

backend/tests/test_maintenance.py

@@ -5,7 +5,9 @@ from __future__ import annotations
 import hashlib
 import json
 import sqlite3
+from collections.abc import Callable
 from pathlib import Path
+from typing import Any
 
 import pytest
 from typer.testing import CliRunner
@@ -20,6 +22,7 @@ from ely_eye.memory import MemoryService
 from ely_eye.migrations import SCHEMA_VERSION
 from ely_eye.schemas import EvidenceAtom, MemoryOrigin, Modality, SourceRecord, TrustScores, utc_now
 from ely_eye.storage import ObjectStore
+from ely_eye.vector_index import VectorIndex
 
 KEYS = "the keys are on the shelf by the door"
 PASSPORT = "the passport is in the blue box"
@@ -85,6 +88,29 @@ def test_a_restore_brings_the_memories_back(tmp_path: Path) -> None:
     assert {atom.text for atom in database.list_atoms()} == {KEYS, PASSPORT}
 
 
+def test_a_backup_carries_the_vector_index_and_a_restore_answers_from_it(
+    tmp_path: Path, fake_embeddings: Callable[[], Any]
+) -> None:
+    source = Settings(home=tmp_path / "home", default_holder="wife", dense_embeddings_enabled=True)
+    database = Database(source)
+    database.init()
+    index = VectorIndex(source, database, fake_embeddings())
+    memory = MemoryService(source, database, vectors=index)
+    for statement in (KEYS, PASSPORT):
+        memory.remember(statement, holder_id="wife", origin=MemoryOrigin.self_)
+    assert index.stats()["vectors"] == 2
+
+    maintenance.backup(source, tmp_path / "backup")
+    destination = Settings(home=tmp_path / "elsewhere", default_holder="wife", dense_embeddings_enabled=True)
+    maintenance.restore(destination, tmp_path / "backup")
+
+    restored = VectorIndex(destination, Database(destination), fake_embeddings())
+    assert restored.stats()["vectors"] == 2, "the vectors are data, not a cache to redo"
+    hits = restored.search(restored.embeddings.embed_query(KEYS), 10)
+    best = next(iter(hits))
+    assert Database(destination).get_atoms([best])[0].text == KEYS
+
+
 def test_a_restore_keeps_the_store_it_replaces(tmp_path: Path) -> None:
     settings = _settings(tmp_path)
     _store(settings)

backend/tests/test_memory_write.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
+from typing import Any
 
 import pytest
 
@@ -18,6 +20,7 @@ from ely_eye.schemas import (
     TimePrecision,
     TrustScores,
 )
+from ely_eye.vector_index import VectorIndex
 
 KEYS = "我们的钥匙放在玄关抽屉里"
 NOW = datetime(2026, 5, 1, 12, 0, tzinfo=UTC)
@@ -124,8 +127,8 @@ def test_time_travel_reads_the_memory_that_was_true_then(service: MemoryService)
     }
 
 
-def test_forget_reaches_the_row_the_index_the_edges_the_cache_and_the_vectors(
-    service: MemoryService,
+def test_forget_reaches_every_copy_of_a_memory(
+    service: MemoryService, fake_embeddings: Callable[[], Any]
 ) -> None:
     atom = _seed_atom("seed-1:text", "seed-1")
     _seed(service.db, atom)
@@ -133,23 +136,21 @@ def test_forget_reaches_the_row_the_index_the_edges_the_cache_and_the_vectors(
         [{"from_atom_id": "seed-1:text", "to_atom_id": "other", "relation": "cites", "confidence": 1.0}]
     )
     service.db.log_cache_event(layer="compiler", cache_key="k", event="hit", bytes_estimate=10)
-    vectors = service.settings.cache_dir / "dense_index.npy"
-    vectors.write_bytes(b"stale")
-    (service.settings.cache_dir / "dense_index_atoms.json").write_text("[]", encoding="utf-8")
+    VectorIndex(service.settings, service.db, fake_embeddings()).sync()
 
     report = service.forget(atom_id="seed-1:text", reason="the drawer is empty")
 
     assert report.atoms_deleted == 1
     assert report.edges_deleted == 1
     assert report.cache_entries_cleared == 1
-    assert report.vector_index_dropped is True
+    assert report.vectors_dropped == 1
+    assert VectorIndex(service.settings, service.db, fake_embeddings()).stats()["vectors"] == 0
     assert service.db.atom_count() == 0
     with service.db.connect() as connection:
         assert connection.execute("SELECT COUNT(*) FROM atom_fts").fetchone()[0] == 0
         assert connection.execute("SELECT COUNT(*) FROM evidence_edges").fetchone()[0] == 0
         trace = connection.execute("SELECT layer, event FROM cache_events").fetchall()
     assert [(row["layer"], row["event"]) for row in trace] == [("memory", "invalidated")]
-    assert not vectors.exists()
 
 
 def test_forget_names_the_cartridge_that_exported_the_source(service: MemoryService) -> None:

backend/tests/test_vector_index.py

@@ -0,0 +1,291 @@
+"""The dense index answers from the database, so a memory is findable the moment it is written.
+
+Every test here runs with a stand-in model: what is being proved is the index's freshness and
+its boundaries, not the quality of a 500 MB snapshot.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from ely_eye.config import Settings
+from ely_eye.db import Database
+from ely_eye.memory import MemoryService
+from ely_eye.retrieval import RetrievalService
+from ely_eye.schemas import MemoryOrigin, SearchFilters
+from ely_eye.vector_index import VectorIndex, expected_vectors
+
+KEYS = "我们的钥匙放在玄关抽屉里"
+PASSPORT = "护照在书房第二个抽屉"
+
+
+def _settings(tmp_path: Path, model_id: str = "hash-mini") -> Settings:
+    return Settings(home=tmp_path, dense_embeddings_enabled=True, embedding_model_id=model_id)
+
+
+def _index(settings: Settings, database: Database, fake_embeddings: Callable[[], Any]) -> VectorIndex:
+    return VectorIndex(settings, database, fake_embeddings())
+
+
+def _memory(settings: Settings, database: Database, index: VectorIndex) -> MemoryService:
+    return MemoryService(settings, database, vectors=index)
+
+
+def test_a_memory_written_now_is_findable_now(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    _memory(settings, database, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+
+    hits = index.search(index.embeddings.embed_query(KEYS), 10)
+
+    assert list(hits) == ["mem-" + _digest(KEYS, "wife") + ":text"], "no rebuild step stands between writing and finding"
+
+
+def _digest(statement: str, holder: str) -> str:
+    import hashlib
+
+    return hashlib.sha256(f"{holder}\n{statement}".encode()).hexdigest()[:16]
+
+
+def test_a_closed_memory_stops_answering(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    memory = _memory(settings, database, index)
+    old = memory.remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+
+    assert index.search(index.embeddings.embed_query(KEYS), 10) == {old.atom_id: pytest.approx(1.0)}
+    replacement = memory.supersede(old.atom_id, text=PASSPORT, holder_id="wife")
+
+    after = index.search(index.embeddings.embed_query(KEYS), 10)
+    assert old.atom_id not in after, "a closed memory stops answering"
+    assert set(after) == {replacement.atom_id}
+
+
+def test_a_rewritten_memory_is_encoded_again(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    atom = _memory(settings, database, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    index.sync()
+
+    rewritten = atom.model_copy(update={"text": PASSPORT})
+    database.upsert_atoms([rewritten])
+    index.sync()
+
+    stale = index.search(index.embeddings.embed_query(KEYS), 10)
+    fresh = index.search(index.embeddings.embed_query(PASSPORT), 10)
+    assert stale[rewritten.atom_id] < 0.9, "the old wording still holds the old vector"
+    assert fresh[rewritten.atom_id] > 0.99
+
+
+def test_a_forgetting_leaves_no_vector_behind(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    memory = _memory(settings, database, index)
+    memory.remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    index.sync()
+
+    report = memory.forget(atom_id="mem-" + _digest(KEYS, "wife") + ":text", reason="moved house")
+
+    assert report.vectors_dropped == 1
+    assert index.stats()["vectors"] == 0
+
+
+def test_somebody_elses_memory_is_not_mine_to_find(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    memory = _memory(settings, database, index)
+    hers_atom = memory.remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    his_atom = memory.remember(PASSPORT, holder_id="husband", origin=MemoryOrigin.self_)
+    index.sync()
+
+    hers = index.search(index.embeddings.embed_query(KEYS), 10, as_holder="wife")
+    his = index.search(index.embeddings.embed_query(KEYS), 10, as_holder="husband")
+
+    assert set(hers) == {hers_atom.atom_id}
+    assert set(his) == {his_atom.atom_id}, "he answers from his own memory, not from hers"
+
+
+def test_a_shared_memory_is_findable_by_both(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    from ely_eye.schemas import Visibility
+
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    memory = _memory(settings, database, index)
+    memory.remember(
+        KEYS, holder_id="wife", origin=MemoryOrigin.self_, visibility=Visibility(scope="visible_to", holders=["husband"])
+    )
+    index.sync()
+
+    assert index.search(index.embeddings.embed_query(KEYS), 10, as_holder="wife")
+    assert index.search(index.embeddings.embed_query(KEYS), 10, as_holder="husband")
+    assert index.search(index.embeddings.embed_query(KEYS), 10, as_holder="daughter") == {}
+
+
+def test_a_cartridge_scope_cannot_be_walked_around(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    from datetime import UTC, datetime
+
+    from ely_eye import cartridge_store
+
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    memory = _memory(settings, database, index)
+    kept = memory.remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    other = memory.remember(PASSPORT, holder_id="wife", origin=MemoryOrigin.self_)
+    index.sync()
+    database.write_cartridge(
+        cartridge_id="cart-1",
+        name="household",
+        root_path=tmp_path / "cart-1",
+        manifest_json="{}",
+        dna="d" * 64,
+        token_equivalent=1,
+        created_at=datetime(2026, 5, 1, tzinfo=UTC),
+    )
+    cartridge_store.add_membership(database, "cart-1", [kept.atom_id])
+
+    hits = index.search(index.embeddings.embed_query(KEYS + " " + PASSPORT), 10, cartridge_ids=["cart-1"])
+
+    assert set(hits) == {kept.atom_id}, "the scope is applied inside the engine, not after it"
+    assert other.atom_id not in hits
+
+
+def test_a_different_model_rebuilds_instead_of_mixing_vectors(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    _memory(settings, database, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    index.sync()
+    assert index.stats()["vectors"] == 1
+
+    swapped = VectorIndex(_settings(tmp_path, model_id="other-model"), database, fake_embeddings())
+    swapped.sync()
+
+    facts = swapped.stats()
+    assert facts["model"] == "other-model"
+    assert facts["vectors"] == 1, "the table was rebuilt, not appended to"
+
+
+def test_a_picture_with_no_words_is_the_other_indexs_job(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    from datetime import UTC, datetime
+
+    from ely_eye.schemas import EvidenceAtom, Modality, SourceRecord, TrustScores
+
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    now = datetime(2026, 5, 1, tzinfo=UTC)
+    database.upsert_sources(
+        [
+            SourceRecord(
+                source_id="pic-1",
+                path="/tmp/pic.png",
+                kind="image",
+                mime="image/png",
+                sha256="0" * 64,
+                size_bytes=4,
+                created_at=now,
+            )
+        ]
+    )
+    database.upsert_atoms(
+        [
+            EvidenceAtom(
+                atom_id="pic-1:image",
+                modality=Modality.image,
+                source_id="pic-1",
+                source="pic-1",
+                ingested_at=now,
+                text="",
+                image_ref="pic-1",
+                trust=TrustScores(parser=1.0),
+                token_equivalent=1,
+                holder_id="wife",
+            )
+        ]
+    )
+
+    assert index.sync()["encoded"] == 0
+    assert expected_vectors(database) == 0
+    assert index.stats()["vectors"] == 0, "a wordless picture is the visual index's job"
+
+
+def test_the_index_survives_a_reopen(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings = _settings(tmp_path)
+    first = Database(settings)
+    index = _index(settings, first, fake_embeddings)
+    _memory(settings, first, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    index.sync()
+
+    reopened = VectorIndex(settings, Database(settings), fake_embeddings())
+
+    assert reopened.stats()["vectors"] == 1
+    assert reopened.search(reopened.embeddings.embed_query(KEYS), 10)
+
+
+def test_superseding_moves_the_vector_rather_than_keeping_two(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    memory = _memory(settings, database, index)
+    old = memory.remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+
+    memory.supersede(old.atom_id, text=PASSPORT, holder_id="wife")
+
+    facts = index.stats()
+    assert facts["vectors"] == 1, "the closed memory lost its vector and the replacement took its place"
+    assert facts["pending"] == 0
+
+
+def test_a_store_that_never_built_its_index_says_so_instead_of_stalling_a_search(
+    tmp_path: Path, fake_embeddings: Callable[[], Any]
+) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    atom = _memory(settings, database, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    index.drop_atoms([atom.atom_id])
+
+    facts = index.stats()
+    assert (facts["vectors"], facts["expected"], facts["pending"]) == (0, 1, 1)
+    assert index.search(index.embeddings.embed_query(KEYS), 10) == {}, "a search never rebuilds"
+    assert index.sync()["encoded"] == 1
+    assert index.search(index.embeddings.embed_query(KEYS), 10)
+
+
+def test_the_dense_leg_hides_what_the_view_hides(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    """The search repeats the view's currency rules, so the two must keep answering alike."""
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    atom = _memory(settings, database, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+    lapsed = (datetime.now(UTC) - timedelta(days=1)).isoformat()
+    with database.connect() as connection:
+        connection.execute("UPDATE atoms SET retention_until = ? WHERE atom_id = ?", (lapsed, atom.atom_id))
+        visible = [row["atom_id"] for row in connection.execute("SELECT atom_id FROM v_atoms_current")]
+
+    assert atom.atom_id not in visible, "the view has retired the memory"
+    assert index.stats()["vectors"] == 1, "the index still holds its vector"
+    assert index.search(index.embeddings.embed_query(KEYS), 10) == {}, "a search retires it too"
+
+
+def test_a_query_asks_for_a_real_number_of_answers(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    index = _index(settings, database, fake_embeddings)
+    with pytest.raises(ValueError, match="positive limit"):
+        index.search(index.embeddings.embed_query(KEYS), 0)
+
+
+def test_search_answers_from_the_index_without_a_rebuild(tmp_path: Path, fake_embeddings: Callable[[], Any]) -> None:
+    settings, database = _settings(tmp_path), Database(_settings(tmp_path))
+    embeddings = fake_embeddings()
+    service = RetrievalService(settings, database)
+    service.embeddings = embeddings  # type: ignore[assignment]
+    index = VectorIndex(settings, database, embeddings)
+    service.vectors = index
+    _memory(settings, database, index).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_)
+
+    result = service.search(KEYS, filters=SearchFilters(as_holder="wife"))
+
+    assert result.hits, "the needle was written and found in the same breath"
+    assert result.hits[0].atom.atom_id == "mem-" + _digest(KEYS, "wife") + ":text"
+    assert result.hits[0].dense_score > 0.0

backend/tests/test_visual_index.py

@@ -180,7 +180,7 @@ def test_forgetting_a_picture_voids_the_picture_index(tmp_path: Path) -> None:
 
     report = MemoryService(settings, database).forget(source_id=atom.source_id)
 
-    assert report.vector_index_dropped
+    assert report.visual_index_dropped
     assert not index.index_path.exists() and not index.map_path.exists()
     assert index.load() is None, "a forgotten picture must not keep answering from memory"