ZacharyZhang-NY
fc0698c242
- bench/ingest_baseline.py drives the unmodified product IngestionService over all 50,789 corpus files in sharded SQLite DBs, then merges them. Thread and memory ceilings are set explicitly: ONNX Runtime defaulted to every core (95 threads/worker, load 490) and its CPU arena kept freed pages mapped, which OOM-killed workers. - bench/metrics.py: recall@k, MRR, nDCG@10, Wilson lower bound, chance baseline. No model, no product import, so the scorer cannot grade itself. - bench/systems.py + bench/run.py: the five ablation arms over the merged DB, latency percentiles, scale curve, and a top-10 equivalence check against the live RetrievalService. - bench/counterpart.py + bench/run_external.py: in-repo Mem0-style flat store and Zep-style temporal store as the external reference point. - bench/audit.py: enforces single-attempt scoring, frozen n, a chance row beside every score, and producer != verifier. It already flags the product's retry-until-target loop at backend/src/ely_eye/evals.py:2351. - backend/pyproject.toml: rapidocr-onnxruntime>=1.4.4 was unsatisfiable (PyPI maxes at 1.2.3); the successor is the renamed rapidocr package.
140 lines
5.0 KiB
Python
140 lines
5.0 KiB
Python
"""Checks for the arm plumbing, so a broken ranking cannot hide behind a green baseline run."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from bench.systems import (
|
|
ARMS,
|
|
ArmRunner,
|
|
Atom,
|
|
_unit,
|
|
arm_needs,
|
|
fuse,
|
|
load_atoms,
|
|
local_model_dir,
|
|
tokenize,
|
|
top_sources,
|
|
)
|
|
|
|
ATOMS = [
|
|
Atom(atom_id="a1", source_id="src_red", text="red door key"),
|
|
Atom(atom_id="a2", source_id="src_blue", text="blue drawer"),
|
|
Atom(atom_id="a3", source_id="src_red", text="the key hangs on the red door"),
|
|
]
|
|
|
|
|
|
def test_tokenize_keeps_cjk_and_words() -> None:
|
|
assert tokenize("钥匙 Key") == ["钥匙", "key"]
|
|
assert tokenize("") == []
|
|
|
|
|
|
def test_top_sources_ranks_by_the_best_atom_of_each_source() -> None:
|
|
scores = np.array([1.0, 5.0, 9.0])
|
|
assert top_sources(scores, ATOMS, 10) == ["src_red", "src_blue"]
|
|
|
|
|
|
def test_top_sources_breaks_ties_on_source_id() -> None:
|
|
scores = np.array([3.0, 3.0, 3.0])
|
|
assert top_sources(scores, ATOMS, 10) == ["src_blue", "src_red"]
|
|
|
|
|
|
def test_top_sources_rejects_a_length_mismatch() -> None:
|
|
with pytest.raises(SystemExit, match="scores has 2 entries for 3 atoms"):
|
|
top_sources(np.array([1.0, 2.0]), ATOMS, 3)
|
|
|
|
|
|
def test_unit_scales_to_zero_one_and_flattens_a_constant() -> None:
|
|
np.testing.assert_allclose(_unit(np.array([0.0, 2.0])), [0.0, 1.0])
|
|
np.testing.assert_allclose(_unit(np.array([7.0, 7.0, 7.0])), [0.0, 0.0, 0.0])
|
|
|
|
|
|
def test_fuse_uses_the_product_weights() -> None:
|
|
fused = fuse(np.array([1.0, 0.0]), np.array([0.0, 1.0]), np.array([0.0, 0.0]))
|
|
assert fused[0] == pytest.approx(0.55)
|
|
assert fused[1] == pytest.approx(0.35)
|
|
assert sum([0.55, 0.35, 0.10]) == pytest.approx(1.0)
|
|
|
|
|
|
def test_graph_only_arm_is_a_constant_and_sorts_by_source_id() -> None:
|
|
runner = ArmRunner("graph_only", ATOMS, Path("/nonexistent"))
|
|
assert runner.rank("key") == ["src_blue", "src_red"]
|
|
|
|
|
|
def test_bm25_arm_finds_the_atom_that_names_the_query() -> None:
|
|
runner = ArmRunner("bm25_only", ATOMS, Path("/nonexistent"))
|
|
assert runner.rank("the key hangs")[0] == "src_red"
|
|
# A query with no tokens has no signal: every score is zero, so the order is source id.
|
|
assert runner.rank("") == ["src_blue", "src_red"]
|
|
|
|
|
|
def test_arm_needs_reports_missing_models_without_crashing(tmp_path: Path) -> None:
|
|
assert arm_needs("bm25_only", tmp_path) is True
|
|
assert arm_needs("dense_only", tmp_path) is False
|
|
(local_model_dir(tmp_path, "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")).mkdir(parents=True)
|
|
assert arm_needs("dense_only", tmp_path) is True
|
|
assert arm_needs("fusion_rerank", tmp_path) is False
|
|
|
|
|
|
def test_arm_needs_rejects_an_unknown_arm() -> None:
|
|
with pytest.raises(SystemExit, match="unknown arm"):
|
|
arm_needs("quantum", Path("/nonexistent"))
|
|
assert "fusion_rerank" in ARMS
|
|
|
|
|
|
def test_local_model_dir_flattens_the_namespace() -> None:
|
|
assert local_model_dir(Path("/m"), "BAAI/bge-reranker-base") == Path("/m/bge-reranker-base")
|
|
|
|
|
|
def make_database(path: Path) -> None:
|
|
"""Create the product's atoms table and put three rows in it."""
|
|
connection = sqlite3.connect(path)
|
|
connection.execute(
|
|
"CREATE TABLE atoms (atom_id text PRIMARY KEY, source_id text, modality text, source text, time text,"
|
|
" text text, image_ref text, embedding_refs_json text, layout_json text, relations_json text,"
|
|
" trust_json text, token_equivalent integer, metadata_json text)"
|
|
)
|
|
for atom in ATOMS:
|
|
connection.execute("INSERT INTO atoms (atom_id, source_id, text) VALUES (?, ?, ?)", (atom.atom_id, atom.source_id, atom.text))
|
|
connection.commit()
|
|
connection.close()
|
|
|
|
|
|
def test_load_atoms_returns_every_row_in_a_stable_order(tmp_path: Path) -> None:
|
|
db = tmp_path / "ely_eye.sqlite"
|
|
make_database(db)
|
|
loaded = load_atoms(db)
|
|
assert [atom.atom_id for atom in loaded] == ["a1", "a2", "a3"]
|
|
assert loaded[2].text == "the key hangs on the red door"
|
|
|
|
|
|
def test_load_atoms_fails_fast_on_a_missing_or_unusable_database(tmp_path: Path) -> None:
|
|
with pytest.raises(SystemExit, match="no merged baseline database"):
|
|
load_atoms(tmp_path / "absent.sqlite")
|
|
no_table = tmp_path / "no_table.sqlite"
|
|
no_table.touch()
|
|
with pytest.raises(SystemExit, match="cannot read the atoms table"):
|
|
load_atoms(no_table)
|
|
empty = tmp_path / "empty.sqlite"
|
|
make_database(empty)
|
|
connection = sqlite3.connect(empty)
|
|
connection.execute("DELETE FROM atoms")
|
|
connection.commit()
|
|
connection.close()
|
|
with pytest.raises(SystemExit, match="atoms table is empty"):
|
|
load_atoms(empty)
|
|
|
|
|
|
def test_percentile_needs_timings_and_a_valid_fraction() -> None:
|
|
runner = ArmRunner("graph_only", ATOMS, Path("/nonexistent"))
|
|
with pytest.raises(SystemExit, match="no timings recorded"):
|
|
runner.percentile(0.5)
|
|
runner.rank("key")
|
|
with pytest.raises(SystemExit, match="percentile must be within"):
|
|
runner.percentile(1.5)
|
|
assert runner.percentile(0.5) >= 0.0
|