ZacharyZhang-NY
4ab36cd5eb
The suite had fifteen checks that could not lose. The visual HashHop arena painted a cyan line and a white ring on the answer slot, then asked the model to find the answer by looking; the collector re-rolled puzzles until the running rate crossed its target; the cache check passed if any layer had ever recorded an entry; the repository check passed on a call graph built in memory without touching the store; the research-module check asserted non-empty lists of SHA256-derived placeholder vectors and so could not fail. Each check now answers for one claim and reports a verdict the reader can act on: passed, failed, deferred, or removed. A deferred row names what is missing and the command that would settle it; a removed row says why the old claim was withdrawn. The suite verdict is failed > deferred > passed, so a claim that never ran holds the whole report back from saying "proven". - evals.py (2427 lines) becomes the proofs/ package; largest module 445 lines. - proofs/__main__.py deleted: it duplicated PRDProofSuite.latest() with a different glob. proofs/table.py renders; latest-proof-suite is the one face and exits 1 on a failed verdict. - The arena paints nothing. The puzzle id is printed on its tile instead, because removing the ink without it removes the question. - HashHop runs a fixed 8 puzzles per depth instead of re-rolling to target. - The contradiction corpus moves from code to contradiction_scenarios.json; the check stays deferred while the corpus is the one this repo authored. - The cache check compiles a nonce question over a real atom twice and reads the miss to hit movement; an empty store fails instead of agreeing by luck. - The repository check requires rows in the store; the research-module check is removed. Two product bugs surfaced on the way: - text_index.fts_query joined Latin terms with a space, which FTS5 reads as AND, so a question longer than its answer retrieved nothing. Terms are ORed and bm25 rank orders the candidates. - profiles.windows_path_to_wsl prefixed /mnt/ onto an already-POSIX path with an empty drive, producing /mnt//home/... and failing every launcher probe. Verification: 168 passed (pytest backend/tests bench/tests); ruff clean on the proofs package, cli.py, text_index.py, profiles.py, and both test files; mypy clean on 16 files with a fresh cache; pyright reports 0 errors on proofs/ and the test file. Ruff debt elsewhere is unchanged: 19 errors in seven files this commit does not touch, measured against the same files at HEAD.
111 lines
4.5 KiB
Python
111 lines
4.5 KiB
Python
"""Checks for the FTS normalizer, run against a real FTS5 table.
|
|
|
|
String equality alone would pass a normalizer SQLite cannot use, so every acceptance case
|
|
here inserts indexed text and asks the engine the question the user would ask.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
|
|
import pytest
|
|
|
|
from ely_eye.text_index import fts_query, index_text
|
|
|
|
|
|
def _fts(rows: list[str]) -> sqlite3.Connection:
|
|
connection = sqlite3.connect(":memory:")
|
|
connection.execute("CREATE VIRTUAL TABLE fts USING fts5(original UNINDEXED, text, tokenize = 'unicode61')")
|
|
connection.executemany("INSERT INTO fts(original, text) VALUES (?, ?)", [(row, index_text(row)) for row in rows])
|
|
return connection
|
|
|
|
|
|
def _hits(connection: sqlite3.Connection, query: str) -> list[str]:
|
|
return [row[0] for row in connection.execute("SELECT original FROM fts WHERE fts MATCH ?", (fts_query(query),))]
|
|
|
|
|
|
def _ranked(connection: sqlite3.Connection, query: str) -> list[str]:
|
|
"""Matches in bm25 order: the answer that covers more of the question should come first."""
|
|
return [
|
|
row[0]
|
|
for row in connection.execute(
|
|
"SELECT original FROM fts WHERE fts MATCH ? ORDER BY rank", (fts_query(query),)
|
|
)
|
|
]
|
|
|
|
|
|
def test_index_text_splits_a_chinese_sentence_into_characters() -> None:
|
|
assert index_text("我们的钥匙放在玄关抽屉里") == "我 们 的 钥 匙 放 在 玄 关 抽 屉 里"
|
|
|
|
|
|
def test_index_text_splits_identifiers_a_user_cannot_guess_the_casing_of() -> None:
|
|
assert index_text("func SearchAtoms") == "func search atoms"
|
|
assert index_text("def search_atoms(self) -> None:") == "def search atoms self none"
|
|
assert index_text("HTTPServer2 listens on 8765") == "http server 2 listens on 8765"
|
|
|
|
|
|
def test_index_text_folds_full_width_letters_that_search_would_miss() -> None:
|
|
assert index_text("Ｓｅａｒｃｈ Ａｔｏｍｓ") == "search atoms"
|
|
|
|
|
|
def test_fts_query_phrases_cjk_and_prefixes_latin() -> None:
|
|
assert fts_query("钥匙") == '"钥 匙"'
|
|
assert fts_query("search atoms") == '"search"* OR "atoms"*'
|
|
|
|
|
|
def test_a_long_question_reaches_the_answer_that_covers_most_of_its_words() -> None:
|
|
"""AND over every word answers a long question with nothing; bm25 picks the best candidate."""
|
|
connection = _fts(
|
|
[
|
|
"PRD requirements prove long context memory, HashHop proof, and adapter training.",
|
|
"The adapter was trained for 29 steps on 37 samples.",
|
|
"Unrelated notes about the weather.",
|
|
]
|
|
)
|
|
question = "Which Ely-Eye PRD requirements prove long context memory, HashHop, and adapters?"
|
|
hits = [
|
|
row[0]
|
|
for row in connection.execute(
|
|
"SELECT original FROM fts WHERE fts MATCH ? ORDER BY rank", (fts_query(question),)
|
|
)
|
|
]
|
|
|
|
assert hits[0].startswith("PRD requirements prove")
|
|
assert "Unrelated notes about the weather." not in hits
|
|
|
|
|
|
def test_fts_query_refuses_a_query_with_nothing_to_match() -> None:
|
|
for empty in ("", " ", "?!。"):
|
|
with pytest.raises(ValueError, match="needs at least one searchable word"):
|
|
fts_query(empty)
|
|
|
|
|
|
def test_a_chinese_question_finds_the_chinese_memory() -> None:
|
|
connection = _fts(["我们的钥匙放在玄关抽屉里", "The keys are in the kitchen drawer"])
|
|
assert _hits(connection, "钥匙") == ["我们的钥匙放在玄关抽屉里"]
|
|
|
|
|
|
def test_a_cjk_phrase_will_not_match_its_characters_scattered_apart() -> None:
|
|
connection = _fts(["钥匙在抽屉", "匙 在 钥"])
|
|
assert _hits(connection, "钥匙") == ["钥匙在抽屉"]
|
|
|
|
|
|
def test_an_english_query_finds_the_identifier_it_names() -> None:
|
|
connection = _fts(["func SearchAtoms(ctx context.Context)", "func ListCartridges()"])
|
|
assert _hits(connection, "search atoms") == ["func SearchAtoms(ctx context.Context)"]
|
|
|
|
|
|
def test_a_prefix_reaches_the_word_it_grew_into() -> None:
|
|
connection = _fts(["the atom was forgotten", "unrelated text"])
|
|
assert _hits(connection, "atom") == ["the atom was forgotten"]
|
|
assert _hits(connection, "forg") == ["the atom was forgotten"]
|
|
|
|
|
|
def test_a_mixed_query_ranks_the_memory_holding_both_halves_first() -> None:
|
|
"""A query with a Chinese half and an English half must prefer the memory holding both."""
|
|
connection = _fts(["钥匙放在 drawer 里", "钥匙在玄关", "the drawer is empty"])
|
|
ranked = _ranked(connection, "钥匙 drawer")
|
|
|
|
assert ranked[0] == "钥匙放在 drawer 里"
|
|
assert set(ranked) == {"钥匙放在 drawer 里", "钥匙在玄关", "the drawer is empty"}
|