Add baseline harness: ingest, metrics, ablation arms, external counterpart, audit

- bench/ingest_baseline.py drives the unmodified product IngestionService over
  all 50,789 corpus files in sharded SQLite DBs, then merges them. Thread and
  memory ceilings are set explicitly: ONNX Runtime defaulted to every core
  (95 threads/worker, load 490) and its CPU arena kept freed pages mapped,
  which OOM-killed workers.
- bench/metrics.py: recall@k, MRR, nDCG@10, Wilson lower bound, chance
  baseline. No model, no product import, so the scorer cannot grade itself.
- bench/systems.py + bench/run.py: the five ablation arms over the merged DB,
  latency percentiles, scale curve, and a top-10 equivalence check against the
  live RetrievalService.
- bench/counterpart.py + bench/run_external.py: in-repo Mem0-style flat store
  and Zep-style temporal store as the external reference point.
- bench/audit.py: enforces single-attempt scoring, frozen n, a chance row
  beside every score, and producer != verifier. It already flags the product's
  retry-until-target loop at backend/src/ely_eye/evals.py:2351.
- backend/pyproject.toml: rapidocr-onnxruntime>=1.4.4 was unsatisfiable
  (PyPI maxes at 1.2.3); the successor is the renamed rapidocr package.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-06 14:36:02 -04:00
1 parent 77219c930b
commit fc0698c242
27 files changed
+2589 -24

No files matched your search

+102
View File
@@ -0,0 +1,102 @@
"""The auditor must catch planted cheating and stay quiet on honest code."""
from __future__ import annotations
import json
from typing import Any
import pytest
from bench.audit import (
best_of_violations,
frozen_counts,
injected_cases,
report_violations,
scorer_is_independent,
third_party_origins,
)
RETRY = """
def collect(proofs, target=0.90):
while len(proofs) < 3:
proofs.append(run_once())
if pass_rate >= target:
break
"""
HONEST = """
def collect(proofs, limit):
while len(proofs) < limit:
proofs.append(next_case())
"""
FULL = {"meta": {"queries": 5308, "answerable": 2928}, "chance": {"recall_at_10": 0.0002}, "arms": [{"arm": "a", "overall": {"recall_at_10": 0.4, "hit_at_10": 0.6, "hit_at_10_wilson_lower": 0.5}}]}
def test_a_loop_that_rerolls_until_a_score_clears_is_a_violation() -> None:
found = best_of_violations(RETRY, "evals.py")
assert len(found) == 1
assert "retry-until-score" in found[0] and "pass_rate" in found[0]
def test_a_plain_data_loop_is_not_flagged() -> None:
assert best_of_violations(HONEST, "runner.py") == []
def test_an_explicit_best_of_name_is_flagged_wherever_it_appears() -> None:
assert best_of_violations("best_of_8 = max(scores)", "x.py")
assert best_of_violations("def until_pass(self): ...", "x.py")
def test_unparseable_source_fails_loudly() -> None:
with pytest.raises(SystemExit, match="cannot parse"):
best_of_violations("def broken(:", "broken.py")
def test_a_report_without_a_chance_row_is_refused(tmp_path: Any) -> None:
path = tmp_path / "baseline.json"
report = {k: v for k, v in FULL.items() if k != "chance"}
path.write_text(json.dumps(report), encoding="utf-8")
found = report_violations(path, frozen_counts())
assert any("no chance row" in v for v in found)
def test_a_short_run_must_say_which_limit_it_used(tmp_path: Any) -> None:
path = tmp_path / "baseline.json"
path.write_text(json.dumps({**FULL, "meta": {"queries": 12, "answerable": 6}}), encoding="utf-8")
assert any("no --limit is recorded" in v for v in report_violations(path, frozen_counts()))
limited = {**FULL, "meta": {"queries": 12, "answerable": 6, "limit_per_modality": 2}}
path.write_text(json.dumps(limited), encoding="utf-8")
assert report_violations(path, frozen_counts()) == []
def test_a_score_missing_its_wilson_bound_is_refused(tmp_path: Any) -> None:
path = tmp_path / "baseline.json"
report = json.loads(json.dumps(FULL))
del report["arms"][0]["overall"]["hit_at_10_wilson_lower"]
path.write_text(json.dumps(report), encoding="utf-8")
assert any("hit_at_10_wilson_lower" in v for v in report_violations(path, frozen_counts()))
def test_an_unreadable_report_is_a_violation_not_a_crash(tmp_path: Any) -> None:
path = tmp_path / "baseline.json"
path.write_text("{not json", encoding="utf-8")
assert any("unreadable report" in v for v in report_violations(path, frozen_counts()))
def test_the_real_scorer_needs_no_model_and_the_real_gold_is_third_party() -> None:
assert scorer_is_independent() == []
assert third_party_origins() == []
def test_every_planted_case_is_caught(tmp_path: Any) -> None:
manifest = frozen_counts()
for index, (source, expected) in enumerate(injected_cases()):
if expected[0] == "retry-until-score":
found = best_of_violations(source, "injected.py")
else:
path = tmp_path / f"injected_{index}.json"
path.write_text(source, encoding="utf-8")
found = report_violations(path, manifest)
assert found, f"the auditor accepted {expected[0]}"
assert expected[0] in found[0], f"{expected[0]} was caught for the wrong reason: {found[0]}"
+125
View File
@@ -0,0 +1,125 @@
"""Behaviour checks for the memory counterparts: scoping, dedup, and invalidation are the whole point."""
from __future__ import annotations
from pathlib import Path
import pytest
from bench.counterpart import (
Counterpart,
MemoryItem,
SourceMeta,
build_items,
overlap,
split_items,
subject_of,
supersede,
)
from bench.systems import Atom
MODEL_DIR = Path("/nonexistent-model")
def item(text: str, source_id: str, holder: str, observed_at: str, subject: str = "") -> MemoryItem:
return MemoryItem(
item_id=f"{source_id}#0",
text=text,
source_id=source_id,
holder=holder,
scenario="assistant-memory",
observed_at=observed_at,
subject=subject,
valid_from=observed_at,
)
def test_split_items_cuts_at_the_length_limit() -> None:
chunks = split_items("word " * 300)
assert chunks, "a long text must produce items"
assert all(len(chunk) <= 400 for chunk in chunks)
assert split_items(" ") == []
def test_split_items_merges_short_sentences_into_one_item() -> None:
assert len(split_items("The keys are in the drawer. They are blue.")) == 1
def test_subject_of_needs_a_repeated_content_token() -> None:
assert subject_of("the camera camera camera") == "camera"
assert subject_of("a single camera mention") == ""
assert subject_of("") == ""
def test_overlap_separates_a_restatement_from_a_new_statement() -> None:
assert overlap("the keys are in the blue drawer", "the keys are in the blue drawer") == 1.0
assert overlap("the keys are in the blue drawer", "completely unrelated bicycle garage") < 0.5
assert overlap("", "anything") == 0.0
def test_build_items_dedups_within_a_holder_and_keeps_both_across_holders() -> None:
index = {
"src_a": SourceMeta("src_a", "ana", "assistant-memory", "conversation", "2023-01-01T00:00:00+00:00"),
"src_b": SourceMeta("src_b", "bo", "assistant-memory", "conversation", "2023-01-02T00:00:00+00:00"),
}
atoms = [
Atom(atom_id="a1", source_id="src_a", text="The keys hang on the red door."),
Atom(atom_id="a2", source_id="src_a", text="The keys hang on the red door."),
Atom(atom_id="b1", source_id="src_b", text="The keys hang on the red door."),
]
items = build_items(atoms, index)
assert len(items) == 2
assert {i.holder for i in items} == {"ana", "bo"}
assert all(i.item_id.startswith(f"{i.source_id}#") for i in items)
def test_supersede_closes_the_older_statement_about_the_same_subject() -> None:
older = item("my camera collection is in the attic", "src_1", "ana", "2023-01-01T00:00:00+00:00", "camera")
newer = item("the bicycle is parked in the garage now", "src_2", "ana", "2023-06-01T00:00:00+00:00", "camera")
assert supersede([older, newer]) == 1
assert older.valid_to == "2023-06-01T00:00:00+00:00"
assert newer.valid_to is None
def test_supersede_leaves_a_restatement_alone() -> None:
first = item("the keys hang on the red door", "src_1", "ana", "2023-01-01T00:00:00+00:00", "keys")
again = item("the keys hang on the red door too", "src_2", "ana", "2023-02-01T00:00:00+00:00", "keys")
assert supersede([first, again]) == 0
assert first.valid_to is None
def test_supersede_is_scoped_to_one_holder() -> None:
ana = item("my camera collection is in the attic", "src_1", "ana", "2023-01-01T00:00:00+00:00", "camera")
bo = item("the bicycle is parked in the garage now", "src_2", "bo", "2023-06-01T00:00:00+00:00", "camera")
assert supersede([ana, bo]) == 0
assert ana.valid_to is None
def test_a_query_only_sees_its_own_holders_items() -> None:
ana = item("the keys hang on the red door", "src_ana", "ana", "2023-01-01T00:00:00+00:00")
bo = item("the keys hang on the green door", "src_bo", "bo", "2023-01-01T00:00:00+00:00")
store = Counterpart(name="mem0_flat", temporal=False, items=[ana, bo], model_dir=MODEL_DIR)
mask = store.visible("ana", None)
assert [store.items[i].holder for i, seen in enumerate(mask) if seen] == ["ana"]
def test_the_temporal_store_hides_closed_items_and_the_flat_store_does_not() -> None:
closed = item("the keys hang on the red door", "src_old", "ana", "2023-01-01T00:00:00+00:00")
closed.valid_to = "2023-06-01T00:00:00+00:00"
open_item = item("the keys hang on the green door", "src_new", "ana", "2023-06-01T00:00:00+00:00")
items = [closed, open_item]
flat = Counterpart(name="mem0_flat", temporal=False, items=items, model_dir=MODEL_DIR)
temporal = Counterpart(name="zep_temporal", temporal=True, items=items, model_dir=MODEL_DIR)
as_of = "2023-12-01T00:00:00+00:00"
assert sum(flat.visible("ana", as_of)) == 2
assert [store.source_id for i, store in enumerate(temporal.items) if temporal.visible("ana", as_of)[i]] == [
"src_new"
]
before = "2023-02-01T00:00:00+00:00"
assert sum(temporal.visible("ana", before)) == 1
def test_percentile_needs_timings() -> None:
store = Counterpart(name="mem0_flat", temporal=False, items=[], model_dir=MODEL_DIR)
with pytest.raises(SystemExit, match="no timings recorded"):
store.percentile(0.5)
+103
View File
@@ -0,0 +1,103 @@
"""Hand-computed checks for every metric, so a wrong number cannot hide behind a green run."""
from __future__ import annotations
import pytest
from bench.metrics import (
aggregate,
chance_recall_at_k,
first_hit_rank,
ndcg_at_10,
recall_at_k,
reciprocal_rank,
score_query,
wilson_lower_bound,
)
def test_recall_at_k_counts_distinct_gold_in_window() -> None:
assert recall_at_k(["a", "b", "c", "d"], {"c"}, 1) == 0.0
assert recall_at_k(["a", "b", "c", "d"], {"c"}, 3) == 1.0
assert recall_at_k(["c", "x", "a"], {"a", "c"}, 1) == 0.5
assert recall_at_k(["c", "x", "a"], {"a", "c"}, 3) == 1.0
assert recall_at_k(["a", "a", "b"], {"a", "b"}, 3) == 1.0 # duplicates count once
def test_recall_is_zero_when_gold_is_empty() -> None:
assert recall_at_k(["a"], set(), 3) == 0.0
def test_reciprocal_rank_uses_first_hit_only() -> None:
assert reciprocal_rank(["x", "a", "y", "b"], {"b"}) == 0.25
assert reciprocal_rank(["x", "a", "y", "b"], {"a", "b"}) == 0.5
assert reciprocal_rank(["x", "y"], {"z"}) == 0.0
assert first_hit_rank(["x", "a", "y", "b"], {"b"}) == 4
assert first_hit_rank(["x", "y"], {"z"}) is None
def test_ndcg_matches_hand_computation() -> None:
# DCG = 1/log2(2) + 1/log2(4) = 1.5; IDCG = 1/log2(2) + 1/log2(3) = 1.6309297535714574
assert ndcg_at_10(["a", "x", "b"], {"a", "b"}) == pytest.approx(1.5 / 1.6309297535714574)
assert ndcg_at_10(["a", "b"], {"a", "b"}) == pytest.approx(1.0)
assert ndcg_at_10(["x", "y"], {"a"}) == 0.0
def test_ndcg_ignores_results_past_ten() -> None:
ranked = ["x"] * 10 + ["a"]
assert ndcg_at_10(ranked, {"a"}) == 0.0
def test_wilson_lower_bound_matches_published_interval() -> None:
# 90/100 at 95% two-sided is [0.8257, 0.9456] in standard tables.
assert wilson_lower_bound(90, 100) == pytest.approx(0.8257, abs=1e-3)
assert wilson_lower_bound(0, 10) == 0.0
assert wilson_lower_bound(10, 10) == pytest.approx(0.7225, abs=1e-3)
assert wilson_lower_bound(1, 10) < 0.5
def test_wilson_rejects_impossible_counts() -> None:
with pytest.raises(SystemExit):
wilson_lower_bound(5, 0)
with pytest.raises(SystemExit):
wilson_lower_bound(11, 10)
def test_chance_recall_is_uniform_baseline() -> None:
assert chance_recall_at_k(1, 1000, 10) == pytest.approx(0.01)
assert chance_recall_at_k(4, 100, 10) == pytest.approx(0.1)
assert chance_recall_at_k(0, 100, 10) == 0.0
with pytest.raises(SystemExit):
chance_recall_at_k(1, 0, 10)
def test_score_query_records_holder_and_abstention() -> None:
holder_of = {"a": "alice", "b": "bob"}
scored = score_query("q1", ["b"], {"b"}, holder="alice", holder_of=holder_of)
assert scored.gold == 1
assert scored.recall_at_1 == 1.0
assert scored.abstained is False
assert scored.top_hit_from_holder is False
assert score_query("q2", [], {"b"}, holder="alice", holder_of=holder_of).abstained is True
def test_aggregate_separates_answerable_from_unanswerable() -> None:
scores = [
score_query("hit", ["a", "x", "y"], {"a"}, holder="alice"),
score_query("miss", ["x", "y", "a"], {"a"}, holder="alice"),
score_query("none", [], set(), holder="alice"),
]
summary = aggregate(scores)
assert summary["n_answerable"] == 2
assert summary["n_unanswerable"] == 1
assert summary["recall_at_1"] == pytest.approx(0.5)
assert summary["recall_at_10"] == pytest.approx(1.0)
assert summary["mrr"] == pytest.approx((1.0 + 1 / 3) / 2)
assert summary["hit_at_10"] == 1.0
assert summary["abstention_rate_answerable"] == 0.0
assert summary["abstention_rate_unanswerable"] == 1.0
def test_aggregate_needs_answerable_queries() -> None:
with pytest.raises(SystemExit):
aggregate([score_query("none", [], set())])
+139
View File
@@ -0,0 +1,139 @@
"""Checks for the arm plumbing, so a broken ranking cannot hide behind a green baseline run."""
from __future__ import annotations
import sqlite3
from pathlib import Path
import numpy as np
import pytest
from bench.systems import (
ARMS,
ArmRunner,
Atom,
_unit,
arm_needs,
fuse,
load_atoms,
local_model_dir,
tokenize,
top_sources,
)
ATOMS = [
Atom(atom_id="a1", source_id="src_red", text="red door key"),
Atom(atom_id="a2", source_id="src_blue", text="blue drawer"),
Atom(atom_id="a3", source_id="src_red", text="the key hangs on the red door"),
]
def test_tokenize_keeps_cjk_and_words() -> None:
assert tokenize("钥匙 Key") == ["钥匙", "key"]
assert tokenize("") == []
def test_top_sources_ranks_by_the_best_atom_of_each_source() -> None:
scores = np.array([1.0, 5.0, 9.0])
assert top_sources(scores, ATOMS, 10) == ["src_red", "src_blue"]
def test_top_sources_breaks_ties_on_source_id() -> None:
scores = np.array([3.0, 3.0, 3.0])
assert top_sources(scores, ATOMS, 10) == ["src_blue", "src_red"]
def test_top_sources_rejects_a_length_mismatch() -> None:
with pytest.raises(SystemExit, match="scores has 2 entries for 3 atoms"):
top_sources(np.array([1.0, 2.0]), ATOMS, 3)
def test_unit_scales_to_zero_one_and_flattens_a_constant() -> None:
np.testing.assert_allclose(_unit(np.array([0.0, 2.0])), [0.0, 1.0])
np.testing.assert_allclose(_unit(np.array([7.0, 7.0, 7.0])), [0.0, 0.0, 0.0])
def test_fuse_uses_the_product_weights() -> None:
fused = fuse(np.array([1.0, 0.0]), np.array([0.0, 1.0]), np.array([0.0, 0.0]))
assert fused[0] == pytest.approx(0.55)
assert fused[1] == pytest.approx(0.35)
assert sum([0.55, 0.35, 0.10]) == pytest.approx(1.0)
def test_graph_only_arm_is_a_constant_and_sorts_by_source_id() -> None:
runner = ArmRunner("graph_only", ATOMS, Path("/nonexistent"))
assert runner.rank("key") == ["src_blue", "src_red"]
def test_bm25_arm_finds_the_atom_that_names_the_query() -> None:
runner = ArmRunner("bm25_only", ATOMS, Path("/nonexistent"))
assert runner.rank("the key hangs")[0] == "src_red"
# A query with no tokens has no signal: every score is zero, so the order is source id.
assert runner.rank("") == ["src_blue", "src_red"]
def test_arm_needs_reports_missing_models_without_crashing(tmp_path: Path) -> None:
assert arm_needs("bm25_only", tmp_path) is True
assert arm_needs("dense_only", tmp_path) is False
(local_model_dir(tmp_path, "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")).mkdir(parents=True)
assert arm_needs("dense_only", tmp_path) is True
assert arm_needs("fusion_rerank", tmp_path) is False
def test_arm_needs_rejects_an_unknown_arm() -> None:
with pytest.raises(SystemExit, match="unknown arm"):
arm_needs("quantum", Path("/nonexistent"))
assert "fusion_rerank" in ARMS
def test_local_model_dir_flattens_the_namespace() -> None:
assert local_model_dir(Path("/m"), "BAAI/bge-reranker-base") == Path("/m/bge-reranker-base")
def make_database(path: Path) -> None:
"""Create the product's atoms table and put three rows in it."""
connection = sqlite3.connect(path)
connection.execute(
"CREATE TABLE atoms (atom_id text PRIMARY KEY, source_id text, modality text, source text, time text,"
" text text, image_ref text, embedding_refs_json text, layout_json text, relations_json text,"
" trust_json text, token_equivalent integer, metadata_json text)"
)
for atom in ATOMS:
connection.execute("INSERT INTO atoms (atom_id, source_id, text) VALUES (?, ?, ?)", (atom.atom_id, atom.source_id, atom.text))
connection.commit()
connection.close()
def test_load_atoms_returns_every_row_in_a_stable_order(tmp_path: Path) -> None:
db = tmp_path / "ely_eye.sqlite"
make_database(db)
loaded = load_atoms(db)
assert [atom.atom_id for atom in loaded] == ["a1", "a2", "a3"]
assert loaded[2].text == "the key hangs on the red door"
def test_load_atoms_fails_fast_on_a_missing_or_unusable_database(tmp_path: Path) -> None:
with pytest.raises(SystemExit, match="no merged baseline database"):
load_atoms(tmp_path / "absent.sqlite")
no_table = tmp_path / "no_table.sqlite"
no_table.touch()
with pytest.raises(SystemExit, match="cannot read the atoms table"):
load_atoms(no_table)
empty = tmp_path / "empty.sqlite"
make_database(empty)
connection = sqlite3.connect(empty)
connection.execute("DELETE FROM atoms")
connection.commit()
connection.close()
with pytest.raises(SystemExit, match="atoms table is empty"):
load_atoms(empty)
def test_percentile_needs_timings_and_a_valid_fraction() -> None:
runner = ArmRunner("graph_only", ATOMS, Path("/nonexistent"))
with pytest.raises(SystemExit, match="no timings recorded"):
runner.percentile(0.5)
runner.rank("key")
with pytest.raises(SystemExit, match="percentile must be within"):
runner.percentile(1.5)
assert runner.percentile(0.5) >= 0.0