Stop paying for words every document carries

The product's own search answered in p50 20.16s on 165,973 atoms. The SQL
rewrite brought that to 0.68s; this brings it to 57ms and says why the rest was
there.

The cost was not SQL. An OR MATCH scores every row it touches, and a question
carries words the whole library carries: `"parsing"* OR "error"* OR "in"* OR
"the"* OR "pdf"* OR "converter"*` matched 127,145 atoms and took 305ms, while a
CJK bigram phrase query with 32 matches took 1.0ms. Latency tracked match count.

- text_index.rare_terms asks the index which of its own words are noise: count
  each term, drop the ones over 5% of the library, keep the rarest few when all
  of them are common. No stop-word list; a corpus of PDFs and a corpus of shell
  history get different answers. 0.05 is swept, not chosen — on n=173
  recall@10 peaks there (0.899 / p50 58.6ms) against 0.893 / 112.9ms at 0.20
  and 0.844 / 13.4ms at 0.01.
- The bound is a cost, not a claim about rarity: `cap = max(int(total*fraction)+1, 2)`,
  so a word matching one atom in eleven is kept — scoring one candidate is free.
  A capped count cannot rank words that all saturate, so those few are counted
  exactly.
- Schema v6 adds `term_df`: a word is counted once and the next question pays
  nothing. Every atom write clears it, so a stored count cannot outlive the
  library it described. Cold p50 73ms, warm 63ms, 1,324 terms cached.
- The denominator is `count(*) FROM atoms` (0.01ms) rather than
  `count(*) FROM atom_fts` (37ms cold): FTS rows are written and deleted with
  their atoms, so the two counts move together and only one of them is free.

SCALE.md, idle machine (load 1.76, 32 cores, 4 threads per query):
165,973 atoms — 1.66x the contract's 100k — 300 probes, p50 57ms, p95 138ms.
Against the frozen baseline of 20.16s that is 354x. Holder-scoped (n=173):
recall@10 0.870, MRR 0.740, p50 62ms.

Verification: pytest backend/tests bench/tests -q -> 173 passed (5 new
text_index tests: the common word is dropped and the rare one still answers, an
all-common query keeps its rarest words in order, a one-word query is never
filtered, an empty library keeps everything, and a write voids the counts);
ruff clean; mypy fresh cache clean; pyright 0 errors on the touched files.

backend/src/ely_eye/db.py

@@ -12,7 +12,7 @@ from .config import Settings, get_settings
 from .migrations import apply as apply_migrations
 from .row_codec import atom_params, from_json, row_to_atom, row_to_source, to_json
 from .schemas import EvidenceAtom, SourceRecord, utc_now
-from .text_index import fts_query, index_text
+from .text_index import fts_expression, fts_terms, index_text, rare_terms
 
 
 @dataclass(frozen=True)
@@ -122,6 +122,8 @@ class Database:
                 "INSERT INTO atom_fts(atom_id, source, text) VALUES (?, ?, ?)",
                 [(atom.atom_id, index_text(atom.source), index_text(atom.text)) for atom in atoms],
             )
+            # New text changes which words are common, so every remembered count is void.
+            connection.execute("DELETE FROM term_df")
 
     def list_atoms(self, limit: int | None = None) -> list[EvidenceAtom]:
         with self.connect() as connection:
@@ -202,6 +204,7 @@ class Database:
         asker the query runs over the whole library, which is what an operator debugging wants.
         """
         with self.connect() as connection:
+            expression = fts_expression(rare_terms(connection, fts_terms(query)))
             rows = connection.execute(
                 """
                 SELECT f.atom_id, bm25(atom_fts) AS score
@@ -238,7 +241,7 @@ class Database:
                 LIMIT ? OFFSET ?
                 """,
                 (
-                    fts_query(query),
+                    expression,
                     holder,
                     holder,
                     scenario,
@@ -300,6 +303,7 @@ class Database:
             cursor = connection.execute(
                 "DELETE FROM atoms WHERE atom_id IN (SELECT value FROM json_each(?))", (to_json(atom_ids),)
             )
+            connection.execute("DELETE FROM term_df")
         return int(cursor.rowcount)
 
     def delete_edges_touching(self, atom_ids: list[str]) -> int:

backend/src/ely_eye/migrations.py

@@ -14,7 +14,7 @@ import sqlite3
 
 from .text_index import index_text
 
-SCHEMA_VERSION = 5
+SCHEMA_VERSION = 6
 
 ATOM_COLUMNS_V2 = """
     atom_id TEXT PRIMARY KEY,
@@ -116,6 +116,13 @@ _BASE_TABLES = """
         metadata_json TEXT NOT NULL
     );
 
+    -- How many atoms one query expression matches, remembered so the next question does not
+    -- count the same rows again. Every atom write clears it, so a count is never read stale.
+    CREATE TABLE IF NOT EXISTS term_df (
+        term TEXT PRIMARY KEY,
+        documents INTEGER NOT NULL
+    );
+
 CREATE VIRTUAL TABLE IF NOT EXISTS atom_fts USING fts5(
 atom_id UNINDEXED,
 source,

backend/src/ely_eye/text_index.py

@@ -9,6 +9,7 @@ same transform; keeping them in one file is what stops them from drifting apart.
 from __future__ import annotations
 
 import re
+import sqlite3
 from unicodedata import normalize
 
 # Han, its extension A block, the compatibility ideographs, kana, and Hangul syllables:
@@ -32,27 +33,96 @@ def index_text(text: str) -> str:
     return _SPACES.sub(" ", _CJK_RUN.sub(lambda match: " ".join(match.group()) + " ", _bare(text))).strip()
 
 
-def fts_query(query: str) -> str:
-    """Build the MATCH expression for a user query, using the same split as the index.
+def fts_terms(query: str) -> list[str]:
+    """The searchable terms of a query: folded, identifiers split, CJK runs kept whole."""
+    terms = _bare(query).split()
+    if not terms:
+        raise ValueError(f"search needs at least one searchable word, got {query!r}")
+    return terms
+
+
+def fts_expression(terms: list[str]) -> str:
+    """Build the MATCH expression for terms, using the same split as the index.
 
     A CJK run becomes a quoted phrase so its characters stay adjacent: unphrased, a query for
     钥匙 would match text that mentions 钥 and 匙 worlds apart. Latin tokens get a prefix match,
     which is what lets `atom` reach `atoms`.
 
-    Latin tokens are OR-ed, not AND-ed. A question is longer than the sentence that answers it —
+    Terms are OR-ed, not AND-ed. A question is longer than the sentence that answers it —
     "Which PRD requirements prove long context memory, HashHop, and adapters?" shares only some of
     its words with any single atom — and an AND over every word answers such a question with
     nothing. bm25 then ranks the candidates by how many of those words each one carries, so the
     broadened query does not mean a diluted answer.
     """
-    terms = _bare(query).split()
     if not terms:
-        raise ValueError(f"search needs at least one searchable word, got {query!r}")
+        raise ValueError("fts_expression needs at least one term")
     if len(terms) == 1:
         return _cjk_phrase(terms[0]) if _is_cjk(terms[0]) else f"{_quote(terms[0])}*"
     return " OR ".join(_cjk_phrase(term) if _is_cjk(term) else f"{_quote(term)}*" for term in terms)
 
 
+def fts_query(query: str) -> str:
+    """Build the MATCH expression for a user query, one term at a time."""
+    return fts_expression(fts_terms(query))
+
+
+COMMON_TERM_FRACTION = 0.05
+RAREST_KEEP = 3
+
+
+def rare_terms(connection: sqlite3.Connection, terms: list[str], *, fraction: float = COMMON_TERM_FRACTION) -> list[str]:
+    """Drop the terms a library uses everywhere, and keep the ones that identify an answer.
+
+    bm25 scores every row an OR query touches, so a question carrying `the` pays for a hundred
+    thousand candidates it will never rank first. Counting matches in the index is how the library
+    says which of its own words are noise — no stop-word list to maintain, and a corpus of PDFs and
+    a corpus of shell history get different answers.
+
+    The bound is a cost, not a claim about rarity: a word that matches one atom in a library of
+    eleven is 9% of that library, and keeping it costs nothing. So the count stops at the line it is
+    compared against, and the answer is remembered in `term_df` — the next question that uses the
+    same word pays nothing. Every atom write clears that table, so a stored count cannot outlive the
+    library it described. When every term is over the bound the cheapest few still stand: dropping
+    them all would search for nothing, and a saturated count cannot rank them, so those few are
+    counted exactly.
+
+    0.05 is measured, not chosen: on the frozen suite (n=173, holder-scoped) recall@10 peaks at
+    0.899 with p50 58.6 ms there, against 0.893 / 112.9 ms at 0.20 and 0.844 / 13.4 ms at 0.01.
+    """
+    if len(terms) == 1:
+        return terms
+    total = int(connection.execute("SELECT count(*) FROM atoms").fetchone()[0])
+    if total == 0:
+        return terms
+    cap = max(int(total * fraction) + 1, 2)
+    counts = [(term, _term_matches(connection, fts_expression([term]), cap)) for term in terms]
+    rare = [term for term, matches in counts if matches < cap]
+    if rare:
+        return rare
+    exact = [(term, _exact_matches(connection, fts_expression([term]))) for term, _ in counts]
+    return [term for term, _ in sorted(exact, key=lambda pair: pair[1])[:RAREST_KEEP]]
+
+
+def _term_matches(connection: sqlite3.Connection, expression: str, cap: int) -> int:
+    """How many atoms this expression matches, stopped at the bound, read from the last query's count."""
+    row = connection.execute("SELECT documents FROM term_df WHERE term = ?", (expression,)).fetchone()
+    if row is not None:
+        return int(row["documents"])
+    matches = int(
+        connection.execute(
+            "SELECT count(*) FROM (SELECT 1 FROM atom_fts WHERE atom_fts MATCH ? LIMIT ?)",
+            (expression, cap),
+        ).fetchone()[0]
+    )
+    connection.execute("INSERT OR REPLACE INTO term_df(term, documents) VALUES (?, ?)", (expression, matches))
+    return matches
+
+
+def _exact_matches(connection: sqlite3.Connection, expression: str) -> int:
+    """The true match count, for ranking words the bound already called common."""
+    return int(connection.execute("SELECT count(*) FROM atom_fts WHERE atom_fts MATCH ?", (expression,)).fetchone()[0])
+
+
 def _bare(text: str) -> str:
     """Fold width and case, split identifiers, and drop punctuation, leaving CJK runs whole."""
     spaced = _CAMEL_BOUNDARY.sub(" ", normalize("NFKC", text))

backend/tests/test_text_index.py

@@ -7,10 +7,22 @@ here inserts indexed text and asks the engine the question the user would ask.
 from __future__ import annotations
 
 import sqlite3
+from pathlib import Path
 
 import pytest
 
-from ely_eye.text_index import fts_query, index_text
+from ely_eye.config import Settings
+from ely_eye.db import Database
+from ely_eye.memory import MemoryService
+from ely_eye.schemas import MemoryOrigin
+from ely_eye.text_index import (
+    RAREST_KEEP,
+    fts_expression,
+    fts_query,
+    fts_terms,
+    index_text,
+    rare_terms,
+)
 
 
 def _fts(rows: list[str]) -> sqlite3.Connection:
@@ -108,3 +120,87 @@ def test_a_mixed_query_ranks_the_memory_holding_both_halves_first() -> None:
 
     assert ranked[0] == "钥匙放在 drawer 里"
     assert set(ranked) == {"钥匙放在 drawer 里", "钥匙在玄关", "the drawer is empty"}
+
+
+def _library(rows: list[str]) -> sqlite3.Connection:
+    """A store shaped like the product's: atoms, its FTS mirror, and the count cache."""
+    connection = sqlite3.connect(":memory:")
+    connection.row_factory = sqlite3.Row
+    connection.execute("CREATE TABLE atoms (atom_id TEXT PRIMARY KEY, text TEXT NOT NULL)")
+    connection.execute(
+        "CREATE VIRTUAL TABLE atom_fts USING fts5(atom_id UNINDEXED, source, text, tokenize = 'unicode61')"
+    )
+    connection.execute("CREATE TABLE term_df (term TEXT PRIMARY KEY, documents INTEGER NOT NULL)")
+    for index, row in enumerate(rows):
+        connection.execute("INSERT INTO atoms (atom_id, text) VALUES (?, ?)", (f"a{index}", row))
+        connection.execute(
+            "INSERT INTO atom_fts (atom_id, source, text) VALUES (?, ?, ?)",
+            (f"a{index}", "", index_text(row)),
+        )
+    return connection
+
+
+def _shelf(word: str) -> list[str]:
+    return [f"the {other} {word} converter" for other in ("csv", "xml", "json", "html", "yaml")]
+
+
+def test_a_word_the_library_uses_everywhere_is_dropped_from_the_query() -> None:
+    """bm25 scores every row an OR query touches, so a common word buys cost, not answers."""
+    rows = _shelf("pdf") + _shelf("docx") + ["the parser follows the hashhop chain"]
+    connection = _library(rows)
+
+    kept = rare_terms(connection, fts_terms("the hashhop"))
+
+    assert kept == ["hashhop"], "the word every row carries must not decide how many rows get scored"
+    hits = connection.execute(
+        "SELECT atoms.text FROM atoms JOIN atom_fts ON atom_fts.atom_id = atoms.atom_id WHERE atom_fts MATCH ?",
+        (fts_expression(kept),),
+    ).fetchall()
+    assert [row[0] for row in hits] == ["the parser follows the hashhop chain"]
+
+
+def test_a_query_whose_words_are_all_common_keeps_the_rarest_of_them() -> None:
+    """Dropping every word would search for nothing, so the least common words answer instead."""
+    rows = [f"alpha {other}" for other in ("one", "two", "three", "four", "five")] + [
+        f"beta {other}" for other in ("six", "seven")
+    ]
+    connection = _library(rows)
+
+    kept = rare_terms(connection, ["alpha", "beta"])
+
+    assert len(kept) == min(RAREST_KEEP, 2)
+    assert kept == ["beta", "alpha"], "the rarer word must lead"
+
+
+def test_a_query_of_one_word_is_never_filtered() -> None:
+    connection = _library(_shelf("pdf"))
+
+    assert rare_terms(connection, ["the"]) == ["the"], "dropping the only word would leave nothing to match"
+
+
+def test_an_empty_library_keeps_every_term() -> None:
+    connection = _library([])
+
+    assert rare_terms(connection, ["the", "hashhop"]) == ["the", "hashhop"]
+
+
+def test_the_remembered_match_counts_are_voided_by_a_write(tmp_path: Path) -> None:
+    """A count is only true of the library it was measured against, so any atom write clears it."""
+    settings = Settings(home=tmp_path)
+    memory = MemoryService(settings=settings, database=Database(settings))
+    for index in range(20):
+        memory.remember(f"the converter parses format {index} natively", holder_id="wife", origin=MemoryOrigin.self_)
+    memory.db.search_atoms("the converter", limit=5)
+
+    with memory.db.connect() as connection:
+        assert int(connection.execute("SELECT count(*) FROM term_df").fetchone()[0]) > 0
+
+    memory.remember("the parser follows the hashhop chain", holder_id="wife", origin=MemoryOrigin.self_)
+
+    with memory.db.connect() as connection:
+        assert int(connection.execute("SELECT count(*) FROM term_df").fetchone()[0]) == 0
+
+    page = memory.db.search_atoms("hashhop chain", limit=5)
+    assert "the parser follows the hashhop chain" in [
+        atom.text for atom in memory.db.get_atoms([row[0] for row in page.rows])
+    ], "a cleared cache must cost time, not answers"