Add baseline harness: ingest, metrics, ablation arms, external counterpart, audit
- bench/ingest_baseline.py drives the unmodified product IngestionService over all 50,789 corpus files in sharded SQLite DBs, then merges them. Thread and memory ceilings are set explicitly: ONNX Runtime defaulted to every core (95 threads/worker, load 490) and its CPU arena kept freed pages mapped, which OOM-killed workers. - bench/metrics.py: recall@k, MRR, nDCG@10, Wilson lower bound, chance baseline. No model, no product import, so the scorer cannot grade itself. - bench/systems.py + bench/run.py: the five ablation arms over the merged DB, latency percentiles, scale curve, and a top-10 equivalence check against the live RetrievalService. - bench/counterpart.py + bench/run_external.py: in-repo Mem0-style flat store and Zep-style temporal store as the external reference point. - bench/audit.py: enforces single-attempt scoring, frozen n, a chance row beside every score, and producer != verifier. It already flags the product's retry-until-target loop at backend/src/ely_eye/evals.py:2351. - backend/pyproject.toml: rapidocr-onnxruntime>=1.4.4 was unsatisfiable (PyPI maxes at 1.2.3); the successor is the renamed rapidocr package.
This commit is contained in:
1 parent
77219c930b
commit
fc0698c242
27 files changed
+2589
-24
No files matched your search
@@ -0,0 +1,102 @@
|
||||
"""The auditor must catch planted cheating and stay quiet on honest code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from bench.audit import (
|
||||
best_of_violations,
|
||||
frozen_counts,
|
||||
injected_cases,
|
||||
report_violations,
|
||||
scorer_is_independent,
|
||||
third_party_origins,
|
||||
)
|
||||
|
||||
RETRY = """
|
||||
def collect(proofs, target=0.90):
|
||||
while len(proofs) < 3:
|
||||
proofs.append(run_once())
|
||||
if pass_rate >= target:
|
||||
break
|
||||
"""
|
||||
|
||||
HONEST = """
|
||||
def collect(proofs, limit):
|
||||
while len(proofs) < limit:
|
||||
proofs.append(next_case())
|
||||
"""
|
||||
|
||||
FULL = {"meta": {"queries": 5308, "answerable": 2928}, "chance": {"recall_at_10": 0.0002}, "arms": [{"arm": "a", "overall": {"recall_at_10": 0.4, "hit_at_10": 0.6, "hit_at_10_wilson_lower": 0.5}}]}
|
||||
|
||||
|
||||
def test_a_loop_that_rerolls_until_a_score_clears_is_a_violation() -> None:
|
||||
found = best_of_violations(RETRY, "evals.py")
|
||||
assert len(found) == 1
|
||||
assert "retry-until-score" in found[0] and "pass_rate" in found[0]
|
||||
|
||||
|
||||
def test_a_plain_data_loop_is_not_flagged() -> None:
|
||||
assert best_of_violations(HONEST, "runner.py") == []
|
||||
|
||||
|
||||
def test_an_explicit_best_of_name_is_flagged_wherever_it_appears() -> None:
|
||||
assert best_of_violations("best_of_8 = max(scores)", "x.py")
|
||||
assert best_of_violations("def until_pass(self): ...", "x.py")
|
||||
|
||||
|
||||
def test_unparseable_source_fails_loudly() -> None:
|
||||
with pytest.raises(SystemExit, match="cannot parse"):
|
||||
best_of_violations("def broken(:", "broken.py")
|
||||
|
||||
|
||||
def test_a_report_without_a_chance_row_is_refused(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
report = {k: v for k, v in FULL.items() if k != "chance"}
|
||||
path.write_text(json.dumps(report), encoding="utf-8")
|
||||
found = report_violations(path, frozen_counts())
|
||||
assert any("no chance row" in v for v in found)
|
||||
|
||||
|
||||
def test_a_short_run_must_say_which_limit_it_used(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
path.write_text(json.dumps({**FULL, "meta": {"queries": 12, "answerable": 6}}), encoding="utf-8")
|
||||
assert any("no --limit is recorded" in v for v in report_violations(path, frozen_counts()))
|
||||
limited = {**FULL, "meta": {"queries": 12, "answerable": 6, "limit_per_modality": 2}}
|
||||
path.write_text(json.dumps(limited), encoding="utf-8")
|
||||
assert report_violations(path, frozen_counts()) == []
|
||||
|
||||
|
||||
def test_a_score_missing_its_wilson_bound_is_refused(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
report = json.loads(json.dumps(FULL))
|
||||
del report["arms"][0]["overall"]["hit_at_10_wilson_lower"]
|
||||
path.write_text(json.dumps(report), encoding="utf-8")
|
||||
assert any("hit_at_10_wilson_lower" in v for v in report_violations(path, frozen_counts()))
|
||||
|
||||
|
||||
def test_an_unreadable_report_is_a_violation_not_a_crash(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
path.write_text("{not json", encoding="utf-8")
|
||||
assert any("unreadable report" in v for v in report_violations(path, frozen_counts()))
|
||||
|
||||
|
||||
def test_the_real_scorer_needs_no_model_and_the_real_gold_is_third_party() -> None:
|
||||
assert scorer_is_independent() == []
|
||||
assert third_party_origins() == []
|
||||
|
||||
|
||||
def test_every_planted_case_is_caught(tmp_path: Any) -> None:
|
||||
manifest = frozen_counts()
|
||||
for index, (source, expected) in enumerate(injected_cases()):
|
||||
if expected[0] == "retry-until-score":
|
||||
found = best_of_violations(source, "injected.py")
|
||||
else:
|
||||
path = tmp_path / f"injected_{index}.json"
|
||||
path.write_text(source, encoding="utf-8")
|
||||
found = report_violations(path, manifest)
|
||||
assert found, f"the auditor accepted {expected[0]}"
|
||||
assert expected[0] in found[0], f"{expected[0]} was caught for the wrong reason: {found[0]}"
|
||||
Reference in new issue
Block a user