Rank memory search in SQL instead of rebuilding BM25 in Python
The product path pulled every atom into Python and rebuilt a BM25Okapi
index per query: p50 20.16s over 165,973 atoms. It now runs one SQLite
statement with LIMIT/OFFSET.
- text_index.py: one normalizer for both sides of the index. CJK runs are
split per character (unicode61 cannot segment them), camelCase and
separators are split so "search atoms" finds SearchAtoms, latin tokens
get a prefix match. Rows written before this existed hold raw text in
atom_fts, so schema v3 rebuilds the index on upgrade.
- retrieval.py: filters (holder / scenario / source / modality / since /
until / min_trust), a base64 cursor, and reciprocal rank fusion in place
of the 0.55/0.35/0.10 blend whose graph term was always zero.
- An empty result is {status: "no_match"}, not a bare array.
- Go: search_atoms and compile_context rank through POST /api/search. The
second Go tokenizer is deleted; two implementations could drift, which
is what the MCP/HTTP parity requirement forbids.
- db.py passed 500 lines, so the row codec moved to row_codec.py.
- Tooling: installed the repo's own pyright. pyrightconfig.json had no
exclude and swept the downloaded corpus (1171 errors); with excludes the
count is 62, all pre-existing (evals 48, profiles 4, training 4,
ingestion 3, runtime 2, cartridge_assets 1). backend/pyproject.toml
carried a second [tool.pyright] that shadowed the root config; deleted.
- bench/scale_check.py measures the product search at corpus size. The
100k-atom p50 number is not written until it is measured on an idle
machine.
Verification: 112 pytest passed; go build/vet/test clean, gofmt clean;
ruff clean on every touched file; mypy clean on the 14 touched Python
files; repo pyright 62 errors, unchanged per-file ledger.
This commit is contained in:
1 parent
3d792417cf
commit
674c88e84e
29 files changed
+1097
-356
No files matched your search
@@ -222,3 +222,22 @@ def test_atoms_as_of_walks_the_validity_timeline(tmp_path: Path) -> None:
|
||||
assert {atom.atom_id for atom in database.atoms_as_of(NOW - timedelta(days=2))} == {"closed"}
|
||||
assert database.atoms_as_of(NOW, holder="nobody") == []
|
||||
assert {atom.atom_id for atom in database.atoms_as_of(NOW, holder="wife")} == {"live"}
|
||||
|
||||
|
||||
def test_a_second_version_database_rebuilds_its_index_on_upgrade(tmp_path: Path) -> None:
|
||||
"""A second-version store indexed raw text, so 钥匙 could never match; the upgrade fixes the index."""
|
||||
database = _database(tmp_path)
|
||||
_seed_source(database)
|
||||
database.upsert_atoms([_atom("key")])
|
||||
with database.connect() as connection:
|
||||
connection.execute("DELETE FROM atom_fts")
|
||||
connection.execute(
|
||||
"INSERT INTO atom_fts(atom_id, source, text) VALUES ('key', 'notes/key.txt', '钥匙放在玄关抽屉里')"
|
||||
)
|
||||
connection.execute("PRAGMA user_version = 2")
|
||||
assert database.search_fts("钥匙", 5) == [], "the old index really could not answer this query"
|
||||
|
||||
upgraded = _database(tmp_path)
|
||||
assert [atom_id for atom_id, _ in upgraded.search_fts("钥匙", 5)] == ["key"]
|
||||
with upgraded.connect() as connection:
|
||||
assert int(connection.execute("PRAGMA user_version").fetchone()[0]) == SCHEMA_VERSION
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Search over HTTP: the answers a caller can act on, and the reasons it gets none."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import ely_eye.api as api_module
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.memory import MemoryService
|
||||
from ely_eye.schemas import MemoryOrigin
|
||||
|
||||
KEYS = "我们的钥匙放在玄关抽屉里"
|
||||
PASSPORT = "护照在书房第二个抽屉里"
|
||||
WALLET = "钱包在玄关的挂钩上"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> TestClient:
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
monkeypatch.setattr(api_module, "get_settings", lambda: settings)
|
||||
return TestClient(api_module.create_app())
|
||||
|
||||
|
||||
def search(client: TestClient, payload: dict) -> dict:
|
||||
response = client.post("/api/search", json=payload)
|
||||
assert response.status_code in (200, 422), response.text
|
||||
return response.json()
|
||||
|
||||
|
||||
def test_a_chinese_question_finds_the_memory_it_was_written_in(client: TestClient) -> None:
|
||||
client.post("/api/memory", json={"text": KEYS, "scenario": "home"})
|
||||
answer = search(client, {"query": "钥匙"})
|
||||
assert answer["status"] == "hits"
|
||||
assert [hit["atom"]["text"] for hit in answer["hits"]] == [KEYS]
|
||||
|
||||
|
||||
def test_an_empty_corpus_answers_no_match_rather_than_an_empty_array(client: TestClient) -> None:
|
||||
answer = search(client, {"query": "钥匙"})
|
||||
assert answer["hits"] == []
|
||||
assert answer["status"] == "no_match", "an empty answer must say the corpus has nothing"
|
||||
|
||||
|
||||
def test_a_query_with_nothing_searchable_is_refused_with_the_reason(client: TestClient) -> None:
|
||||
response = client.post("/api/search", json={"query": "???"})
|
||||
assert response.status_code == 422
|
||||
assert "needs at least one searchable word" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_the_scenario_filter_hides_memories_from_other_scenarios(client: TestClient) -> None:
|
||||
client.post("/api/memory", json={"text": KEYS, "scenario": "home"})
|
||||
assert search(client, {"query": "钥匙", "filters": {"scenario": "home"}})["status"] == "hits"
|
||||
assert search(client, {"query": "钥匙", "filters": {"scenario": "travel"}})["status"] == "no_match"
|
||||
|
||||
|
||||
def test_a_superseded_memory_stops_answering(client: TestClient) -> None:
|
||||
old = client.post("/api/memory", json={"text": KEYS}).json()
|
||||
client.post("/api/memory/supersede", json={"atom_id": old["atom_id"], "text": PASSPORT})
|
||||
answer = search(client, {"query": "抽屉"})
|
||||
assert [hit["atom"]["text"] for hit in answer["hits"]] == [PASSPORT]
|
||||
|
||||
|
||||
def test_paging_walks_every_hit_without_skipping_one(client: TestClient) -> None:
|
||||
for text in (KEYS, PASSPORT, WALLET, "备用钥匙在车里", "工具箱在阳台的抽屉里"):
|
||||
client.post("/api/memory", json={"text": text})
|
||||
seen: list[str] = []
|
||||
cursor = None
|
||||
for _ in range(10):
|
||||
payload: dict = {"query": "钥匙", "top_k": 2}
|
||||
if cursor:
|
||||
payload["cursor"] = cursor
|
||||
answer = search(client, payload)
|
||||
seen.extend(hit["atom"]["atom_id"] for hit in answer["hits"])
|
||||
cursor = answer["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
assert cursor is None, "the page loop never reached the end"
|
||||
assert len(seen) == len(set(seen)), "a page repeated a memory"
|
||||
assert len(seen) == 2, "only two memories mention 钥匙"
|
||||
|
||||
|
||||
def test_the_holder_filter_answers_only_that_holders_memory(
|
||||
client: TestClient, tmp_path: Path
|
||||
) -> None:
|
||||
client.post("/api/memory", json={"text": KEYS})
|
||||
husband = MemoryService(Settings(home=tmp_path, default_holder="husband"), Database(Settings(home=tmp_path)))
|
||||
husband.remember("我的钥匙在办公室", holder_id="husband", origin=MemoryOrigin.self_)
|
||||
answer = search(client, {"query": "钥匙", "filters": {"holder": "husband"}})
|
||||
assert [hit["atom"]["text"] for hit in answer["hits"]] == ["我的钥匙在办公室"]
|
||||
|
||||
|
||||
def test_the_cli_answers_in_json_a_machine_can_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The three faces rank alike: the CLI reports the same hit the HTTP face returns."""
|
||||
import json
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
import ely_eye.cli as cli_module
|
||||
from ely_eye.cli import app
|
||||
|
||||
settings = Settings(home=tmp_path, default_holder="wife")
|
||||
monkeypatch.setattr(cli_module, "get_settings", lambda: settings)
|
||||
runner = CliRunner()
|
||||
MemoryService(settings, Database(settings)).remember(KEYS, holder_id="wife", origin=MemoryOrigin.self_, scenario="home")
|
||||
|
||||
answer = json.loads(runner.invoke(app, ["search", "钥匙", "--scenario", "home"]).stdout)
|
||||
assert answer["status"] == "hits"
|
||||
assert [hit["atom"]["text"] for hit in answer["hits"]] == [KEYS]
|
||||
|
||||
empty = json.loads(runner.invoke(app, ["search", "钥匙", "--scenario", "travel"]).stdout)
|
||||
assert empty["status"] == "no_match"
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Checks for the FTS normalizer, run against a real FTS5 table.
|
||||
|
||||
String equality alone would pass a normalizer SQLite cannot use, so every acceptance case
|
||||
here inserts indexed text and asks the engine the question the user would ask.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from ely_eye.text_index import fts_query, index_text
|
||||
|
||||
|
||||
def _fts(rows: list[str]) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(":memory:")
|
||||
connection.execute("CREATE VIRTUAL TABLE fts USING fts5(original UNINDEXED, text, tokenize = 'unicode61')")
|
||||
connection.executemany("INSERT INTO fts(original, text) VALUES (?, ?)", [(row, index_text(row)) for row in rows])
|
||||
return connection
|
||||
|
||||
|
||||
def _hits(connection: sqlite3.Connection, query: str) -> list[str]:
|
||||
return [row[0] for row in connection.execute("SELECT original FROM fts WHERE fts MATCH ?", (fts_query(query),))]
|
||||
|
||||
|
||||
def test_index_text_splits_a_chinese_sentence_into_characters() -> None:
|
||||
assert index_text("我们的钥匙放在玄关抽屉里") == "我 们 的 钥 匙 放 在 玄 关 抽 屉 里"
|
||||
|
||||
|
||||
def test_index_text_splits_identifiers_a_user_cannot_guess_the_casing_of() -> None:
|
||||
assert index_text("func SearchAtoms") == "func search atoms"
|
||||
assert index_text("def search_atoms(self) -> None:") == "def search atoms self none"
|
||||
assert index_text("HTTPServer2 listens on 8765") == "http server 2 listens on 8765"
|
||||
|
||||
|
||||
def test_index_text_folds_full_width_letters_that_search_would_miss() -> None:
|
||||
assert index_text("Search Atoms") == "search atoms"
|
||||
|
||||
|
||||
def test_fts_query_phrases_cjk_and_prefixes_latin() -> None:
|
||||
assert fts_query("钥匙") == '"钥 匙"'
|
||||
assert fts_query("search atoms") == '"search"* "atoms"*'
|
||||
|
||||
|
||||
def test_fts_query_refuses_a_query_with_nothing_to_match() -> None:
|
||||
for empty in ("", " ", "?!。"):
|
||||
with pytest.raises(ValueError, match="needs at least one searchable word"):
|
||||
fts_query(empty)
|
||||
|
||||
|
||||
def test_a_chinese_question_finds_the_chinese_memory() -> None:
|
||||
connection = _fts(["我们的钥匙放在玄关抽屉里", "The keys are in the kitchen drawer"])
|
||||
assert _hits(connection, "钥匙") == ["我们的钥匙放在玄关抽屉里"]
|
||||
|
||||
|
||||
def test_a_cjk_phrase_will_not_match_its_characters_scattered_apart() -> None:
|
||||
connection = _fts(["钥匙在抽屉", "匙 在 钥"])
|
||||
assert _hits(connection, "钥匙") == ["钥匙在抽屉"]
|
||||
|
||||
|
||||
def test_an_english_query_finds_the_identifier_it_names() -> None:
|
||||
connection = _fts(["func SearchAtoms(ctx context.Context)", "func ListCartridges()"])
|
||||
assert _hits(connection, "search atoms") == ["func SearchAtoms(ctx context.Context)"]
|
||||
|
||||
|
||||
def test_a_prefix_reaches_the_word_it_grew_into() -> None:
|
||||
connection = _fts(["the atom was forgotten", "unrelated text"])
|
||||
assert _hits(connection, "atom") == ["the atom was forgotten"]
|
||||
assert _hits(connection, "forg") == ["the atom was forgotten"]
|
||||
|
||||
|
||||
def test_a_mixed_query_needs_both_halves_in_the_same_memory() -> None:
|
||||
connection = _fts(["钥匙放在 drawer 里", "钥匙在玄关", "the drawer is empty"])
|
||||
assert _hits(connection, "钥匙 drawer") == ["钥匙放在 drawer 里"]
|
||||
Reference in new issue
Block a user