Add baseline harness: ingest, metrics, ablation arms, external counterpart, audit
- bench/ingest_baseline.py drives the unmodified product IngestionService over all 50,789 corpus files in sharded SQLite DBs, then merges them. Thread and memory ceilings are set explicitly: ONNX Runtime defaulted to every core (95 threads/worker, load 490) and its CPU arena kept freed pages mapped, which OOM-killed workers. - bench/metrics.py: recall@k, MRR, nDCG@10, Wilson lower bound, chance baseline. No model, no product import, so the scorer cannot grade itself. - bench/systems.py + bench/run.py: the five ablation arms over the merged DB, latency percentiles, scale curve, and a top-10 equivalence check against the live RetrievalService. - bench/counterpart.py + bench/run_external.py: in-repo Mem0-style flat store and Zep-style temporal store as the external reference point. - bench/audit.py: enforces single-attempt scoring, frozen n, a chance row beside every score, and producer != verifier. It already flags the product's retry-until-target loop at backend/src/ely_eye/evals.py:2351. - backend/pyproject.toml: rapidocr-onnxruntime>=1.4.4 was unsatisfiable (PyPI maxes at 1.2.3); the successor is the renamed rapidocr package.
This commit is contained in:
1 parent
77219c930b
commit
fc0698c242
27 files changed
+2589
-24
No files matched your search
@@ -0,0 +1,102 @@
|
||||
"""The auditor must catch planted cheating and stay quiet on honest code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from bench.audit import (
|
||||
best_of_violations,
|
||||
frozen_counts,
|
||||
injected_cases,
|
||||
report_violations,
|
||||
scorer_is_independent,
|
||||
third_party_origins,
|
||||
)
|
||||
|
||||
RETRY = """
|
||||
def collect(proofs, target=0.90):
|
||||
while len(proofs) < 3:
|
||||
proofs.append(run_once())
|
||||
if pass_rate >= target:
|
||||
break
|
||||
"""
|
||||
|
||||
HONEST = """
|
||||
def collect(proofs, limit):
|
||||
while len(proofs) < limit:
|
||||
proofs.append(next_case())
|
||||
"""
|
||||
|
||||
FULL = {"meta": {"queries": 5308, "answerable": 2928}, "chance": {"recall_at_10": 0.0002}, "arms": [{"arm": "a", "overall": {"recall_at_10": 0.4, "hit_at_10": 0.6, "hit_at_10_wilson_lower": 0.5}}]}
|
||||
|
||||
|
||||
def test_a_loop_that_rerolls_until_a_score_clears_is_a_violation() -> None:
|
||||
found = best_of_violations(RETRY, "evals.py")
|
||||
assert len(found) == 1
|
||||
assert "retry-until-score" in found[0] and "pass_rate" in found[0]
|
||||
|
||||
|
||||
def test_a_plain_data_loop_is_not_flagged() -> None:
|
||||
assert best_of_violations(HONEST, "runner.py") == []
|
||||
|
||||
|
||||
def test_an_explicit_best_of_name_is_flagged_wherever_it_appears() -> None:
|
||||
assert best_of_violations("best_of_8 = max(scores)", "x.py")
|
||||
assert best_of_violations("def until_pass(self): ...", "x.py")
|
||||
|
||||
|
||||
def test_unparseable_source_fails_loudly() -> None:
|
||||
with pytest.raises(SystemExit, match="cannot parse"):
|
||||
best_of_violations("def broken(:", "broken.py")
|
||||
|
||||
|
||||
def test_a_report_without_a_chance_row_is_refused(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
report = {k: v for k, v in FULL.items() if k != "chance"}
|
||||
path.write_text(json.dumps(report), encoding="utf-8")
|
||||
found = report_violations(path, frozen_counts())
|
||||
assert any("no chance row" in v for v in found)
|
||||
|
||||
|
||||
def test_a_short_run_must_say_which_limit_it_used(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
path.write_text(json.dumps({**FULL, "meta": {"queries": 12, "answerable": 6}}), encoding="utf-8")
|
||||
assert any("no --limit is recorded" in v for v in report_violations(path, frozen_counts()))
|
||||
limited = {**FULL, "meta": {"queries": 12, "answerable": 6, "limit_per_modality": 2}}
|
||||
path.write_text(json.dumps(limited), encoding="utf-8")
|
||||
assert report_violations(path, frozen_counts()) == []
|
||||
|
||||
|
||||
def test_a_score_missing_its_wilson_bound_is_refused(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
report = json.loads(json.dumps(FULL))
|
||||
del report["arms"][0]["overall"]["hit_at_10_wilson_lower"]
|
||||
path.write_text(json.dumps(report), encoding="utf-8")
|
||||
assert any("hit_at_10_wilson_lower" in v for v in report_violations(path, frozen_counts()))
|
||||
|
||||
|
||||
def test_an_unreadable_report_is_a_violation_not_a_crash(tmp_path: Any) -> None:
|
||||
path = tmp_path / "baseline.json"
|
||||
path.write_text("{not json", encoding="utf-8")
|
||||
assert any("unreadable report" in v for v in report_violations(path, frozen_counts()))
|
||||
|
||||
|
||||
def test_the_real_scorer_needs_no_model_and_the_real_gold_is_third_party() -> None:
|
||||
assert scorer_is_independent() == []
|
||||
assert third_party_origins() == []
|
||||
|
||||
|
||||
def test_every_planted_case_is_caught(tmp_path: Any) -> None:
|
||||
manifest = frozen_counts()
|
||||
for index, (source, expected) in enumerate(injected_cases()):
|
||||
if expected[0] == "retry-until-score":
|
||||
found = best_of_violations(source, "injected.py")
|
||||
else:
|
||||
path = tmp_path / f"injected_{index}.json"
|
||||
path.write_text(source, encoding="utf-8")
|
||||
found = report_violations(path, manifest)
|
||||
assert found, f"the auditor accepted {expected[0]}"
|
||||
assert expected[0] in found[0], f"{expected[0]} was caught for the wrong reason: {found[0]}"
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Behaviour checks for the memory counterparts: scoping, dedup, and invalidation are the whole point."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from bench.counterpart import (
|
||||
Counterpart,
|
||||
MemoryItem,
|
||||
SourceMeta,
|
||||
build_items,
|
||||
overlap,
|
||||
split_items,
|
||||
subject_of,
|
||||
supersede,
|
||||
)
|
||||
from bench.systems import Atom
|
||||
|
||||
MODEL_DIR = Path("/nonexistent-model")
|
||||
|
||||
|
||||
def item(text: str, source_id: str, holder: str, observed_at: str, subject: str = "") -> MemoryItem:
|
||||
return MemoryItem(
|
||||
item_id=f"{source_id}#0",
|
||||
text=text,
|
||||
source_id=source_id,
|
||||
holder=holder,
|
||||
scenario="assistant-memory",
|
||||
observed_at=observed_at,
|
||||
subject=subject,
|
||||
valid_from=observed_at,
|
||||
)
|
||||
|
||||
|
||||
def test_split_items_cuts_at_the_length_limit() -> None:
|
||||
chunks = split_items("word " * 300)
|
||||
assert chunks, "a long text must produce items"
|
||||
assert all(len(chunk) <= 400 for chunk in chunks)
|
||||
assert split_items(" ") == []
|
||||
|
||||
|
||||
def test_split_items_merges_short_sentences_into_one_item() -> None:
|
||||
assert len(split_items("The keys are in the drawer. They are blue.")) == 1
|
||||
|
||||
|
||||
def test_subject_of_needs_a_repeated_content_token() -> None:
|
||||
assert subject_of("the camera camera camera") == "camera"
|
||||
assert subject_of("a single camera mention") == ""
|
||||
assert subject_of("") == ""
|
||||
|
||||
|
||||
def test_overlap_separates_a_restatement_from_a_new_statement() -> None:
|
||||
assert overlap("the keys are in the blue drawer", "the keys are in the blue drawer") == 1.0
|
||||
assert overlap("the keys are in the blue drawer", "completely unrelated bicycle garage") < 0.5
|
||||
assert overlap("", "anything") == 0.0
|
||||
|
||||
|
||||
def test_build_items_dedups_within_a_holder_and_keeps_both_across_holders() -> None:
|
||||
index = {
|
||||
"src_a": SourceMeta("src_a", "ana", "assistant-memory", "conversation", "2023-01-01T00:00:00+00:00"),
|
||||
"src_b": SourceMeta("src_b", "bo", "assistant-memory", "conversation", "2023-01-02T00:00:00+00:00"),
|
||||
}
|
||||
atoms = [
|
||||
Atom(atom_id="a1", source_id="src_a", text="The keys hang on the red door."),
|
||||
Atom(atom_id="a2", source_id="src_a", text="The keys hang on the red door."),
|
||||
Atom(atom_id="b1", source_id="src_b", text="The keys hang on the red door."),
|
||||
]
|
||||
items = build_items(atoms, index)
|
||||
assert len(items) == 2
|
||||
assert {i.holder for i in items} == {"ana", "bo"}
|
||||
assert all(i.item_id.startswith(f"{i.source_id}#") for i in items)
|
||||
|
||||
|
||||
def test_supersede_closes_the_older_statement_about_the_same_subject() -> None:
|
||||
older = item("my camera collection is in the attic", "src_1", "ana", "2023-01-01T00:00:00+00:00", "camera")
|
||||
newer = item("the bicycle is parked in the garage now", "src_2", "ana", "2023-06-01T00:00:00+00:00", "camera")
|
||||
assert supersede([older, newer]) == 1
|
||||
assert older.valid_to == "2023-06-01T00:00:00+00:00"
|
||||
assert newer.valid_to is None
|
||||
|
||||
|
||||
def test_supersede_leaves_a_restatement_alone() -> None:
|
||||
first = item("the keys hang on the red door", "src_1", "ana", "2023-01-01T00:00:00+00:00", "keys")
|
||||
again = item("the keys hang on the red door too", "src_2", "ana", "2023-02-01T00:00:00+00:00", "keys")
|
||||
assert supersede([first, again]) == 0
|
||||
assert first.valid_to is None
|
||||
|
||||
|
||||
def test_supersede_is_scoped_to_one_holder() -> None:
|
||||
ana = item("my camera collection is in the attic", "src_1", "ana", "2023-01-01T00:00:00+00:00", "camera")
|
||||
bo = item("the bicycle is parked in the garage now", "src_2", "bo", "2023-06-01T00:00:00+00:00", "camera")
|
||||
assert supersede([ana, bo]) == 0
|
||||
assert ana.valid_to is None
|
||||
|
||||
|
||||
def test_a_query_only_sees_its_own_holders_items() -> None:
|
||||
ana = item("the keys hang on the red door", "src_ana", "ana", "2023-01-01T00:00:00+00:00")
|
||||
bo = item("the keys hang on the green door", "src_bo", "bo", "2023-01-01T00:00:00+00:00")
|
||||
store = Counterpart(name="mem0_flat", temporal=False, items=[ana, bo], model_dir=MODEL_DIR)
|
||||
mask = store.visible("ana", None)
|
||||
assert [store.items[i].holder for i, seen in enumerate(mask) if seen] == ["ana"]
|
||||
|
||||
|
||||
def test_the_temporal_store_hides_closed_items_and_the_flat_store_does_not() -> None:
|
||||
closed = item("the keys hang on the red door", "src_old", "ana", "2023-01-01T00:00:00+00:00")
|
||||
closed.valid_to = "2023-06-01T00:00:00+00:00"
|
||||
open_item = item("the keys hang on the green door", "src_new", "ana", "2023-06-01T00:00:00+00:00")
|
||||
items = [closed, open_item]
|
||||
flat = Counterpart(name="mem0_flat", temporal=False, items=items, model_dir=MODEL_DIR)
|
||||
temporal = Counterpart(name="zep_temporal", temporal=True, items=items, model_dir=MODEL_DIR)
|
||||
as_of = "2023-12-01T00:00:00+00:00"
|
||||
assert sum(flat.visible("ana", as_of)) == 2
|
||||
assert [store.source_id for i, store in enumerate(temporal.items) if temporal.visible("ana", as_of)[i]] == [
|
||||
"src_new"
|
||||
]
|
||||
before = "2023-02-01T00:00:00+00:00"
|
||||
assert sum(temporal.visible("ana", before)) == 1
|
||||
|
||||
|
||||
def test_percentile_needs_timings() -> None:
|
||||
store = Counterpart(name="mem0_flat", temporal=False, items=[], model_dir=MODEL_DIR)
|
||||
with pytest.raises(SystemExit, match="no timings recorded"):
|
||||
store.percentile(0.5)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Hand-computed checks for every metric, so a wrong number cannot hide behind a green run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from bench.metrics import (
|
||||
aggregate,
|
||||
chance_recall_at_k,
|
||||
first_hit_rank,
|
||||
ndcg_at_10,
|
||||
recall_at_k,
|
||||
reciprocal_rank,
|
||||
score_query,
|
||||
wilson_lower_bound,
|
||||
)
|
||||
|
||||
|
||||
def test_recall_at_k_counts_distinct_gold_in_window() -> None:
|
||||
assert recall_at_k(["a", "b", "c", "d"], {"c"}, 1) == 0.0
|
||||
assert recall_at_k(["a", "b", "c", "d"], {"c"}, 3) == 1.0
|
||||
assert recall_at_k(["c", "x", "a"], {"a", "c"}, 1) == 0.5
|
||||
assert recall_at_k(["c", "x", "a"], {"a", "c"}, 3) == 1.0
|
||||
assert recall_at_k(["a", "a", "b"], {"a", "b"}, 3) == 1.0 # duplicates count once
|
||||
|
||||
|
||||
def test_recall_is_zero_when_gold_is_empty() -> None:
|
||||
assert recall_at_k(["a"], set(), 3) == 0.0
|
||||
|
||||
|
||||
def test_reciprocal_rank_uses_first_hit_only() -> None:
|
||||
assert reciprocal_rank(["x", "a", "y", "b"], {"b"}) == 0.25
|
||||
assert reciprocal_rank(["x", "a", "y", "b"], {"a", "b"}) == 0.5
|
||||
assert reciprocal_rank(["x", "y"], {"z"}) == 0.0
|
||||
assert first_hit_rank(["x", "a", "y", "b"], {"b"}) == 4
|
||||
assert first_hit_rank(["x", "y"], {"z"}) is None
|
||||
|
||||
|
||||
def test_ndcg_matches_hand_computation() -> None:
|
||||
# DCG = 1/log2(2) + 1/log2(4) = 1.5; IDCG = 1/log2(2) + 1/log2(3) = 1.6309297535714574
|
||||
assert ndcg_at_10(["a", "x", "b"], {"a", "b"}) == pytest.approx(1.5 / 1.6309297535714574)
|
||||
assert ndcg_at_10(["a", "b"], {"a", "b"}) == pytest.approx(1.0)
|
||||
assert ndcg_at_10(["x", "y"], {"a"}) == 0.0
|
||||
|
||||
|
||||
def test_ndcg_ignores_results_past_ten() -> None:
|
||||
ranked = ["x"] * 10 + ["a"]
|
||||
assert ndcg_at_10(ranked, {"a"}) == 0.0
|
||||
|
||||
|
||||
def test_wilson_lower_bound_matches_published_interval() -> None:
|
||||
# 90/100 at 95% two-sided is [0.8257, 0.9456] in standard tables.
|
||||
assert wilson_lower_bound(90, 100) == pytest.approx(0.8257, abs=1e-3)
|
||||
assert wilson_lower_bound(0, 10) == 0.0
|
||||
assert wilson_lower_bound(10, 10) == pytest.approx(0.7225, abs=1e-3)
|
||||
assert wilson_lower_bound(1, 10) < 0.5
|
||||
|
||||
|
||||
def test_wilson_rejects_impossible_counts() -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
wilson_lower_bound(5, 0)
|
||||
with pytest.raises(SystemExit):
|
||||
wilson_lower_bound(11, 10)
|
||||
|
||||
|
||||
def test_chance_recall_is_uniform_baseline() -> None:
|
||||
assert chance_recall_at_k(1, 1000, 10) == pytest.approx(0.01)
|
||||
assert chance_recall_at_k(4, 100, 10) == pytest.approx(0.1)
|
||||
assert chance_recall_at_k(0, 100, 10) == 0.0
|
||||
with pytest.raises(SystemExit):
|
||||
chance_recall_at_k(1, 0, 10)
|
||||
|
||||
|
||||
def test_score_query_records_holder_and_abstention() -> None:
|
||||
holder_of = {"a": "alice", "b": "bob"}
|
||||
scored = score_query("q1", ["b"], {"b"}, holder="alice", holder_of=holder_of)
|
||||
assert scored.gold == 1
|
||||
assert scored.recall_at_1 == 1.0
|
||||
assert scored.abstained is False
|
||||
assert scored.top_hit_from_holder is False
|
||||
assert score_query("q2", [], {"b"}, holder="alice", holder_of=holder_of).abstained is True
|
||||
|
||||
|
||||
def test_aggregate_separates_answerable_from_unanswerable() -> None:
|
||||
scores = [
|
||||
score_query("hit", ["a", "x", "y"], {"a"}, holder="alice"),
|
||||
score_query("miss", ["x", "y", "a"], {"a"}, holder="alice"),
|
||||
score_query("none", [], set(), holder="alice"),
|
||||
]
|
||||
summary = aggregate(scores)
|
||||
assert summary["n_answerable"] == 2
|
||||
assert summary["n_unanswerable"] == 1
|
||||
assert summary["recall_at_1"] == pytest.approx(0.5)
|
||||
assert summary["recall_at_10"] == pytest.approx(1.0)
|
||||
assert summary["mrr"] == pytest.approx((1.0 + 1 / 3) / 2)
|
||||
assert summary["hit_at_10"] == 1.0
|
||||
assert summary["abstention_rate_answerable"] == 0.0
|
||||
assert summary["abstention_rate_unanswerable"] == 1.0
|
||||
|
||||
|
||||
def test_aggregate_needs_answerable_queries() -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
aggregate([score_query("none", [], set())])
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Checks for the arm plumbing, so a broken ranking cannot hide behind a green baseline run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from bench.systems import (
|
||||
ARMS,
|
||||
ArmRunner,
|
||||
Atom,
|
||||
_unit,
|
||||
arm_needs,
|
||||
fuse,
|
||||
load_atoms,
|
||||
local_model_dir,
|
||||
tokenize,
|
||||
top_sources,
|
||||
)
|
||||
|
||||
ATOMS = [
|
||||
Atom(atom_id="a1", source_id="src_red", text="red door key"),
|
||||
Atom(atom_id="a2", source_id="src_blue", text="blue drawer"),
|
||||
Atom(atom_id="a3", source_id="src_red", text="the key hangs on the red door"),
|
||||
]
|
||||
|
||||
|
||||
def test_tokenize_keeps_cjk_and_words() -> None:
|
||||
assert tokenize("钥匙 Key") == ["钥匙", "key"]
|
||||
assert tokenize("") == []
|
||||
|
||||
|
||||
def test_top_sources_ranks_by_the_best_atom_of_each_source() -> None:
|
||||
scores = np.array([1.0, 5.0, 9.0])
|
||||
assert top_sources(scores, ATOMS, 10) == ["src_red", "src_blue"]
|
||||
|
||||
|
||||
def test_top_sources_breaks_ties_on_source_id() -> None:
|
||||
scores = np.array([3.0, 3.0, 3.0])
|
||||
assert top_sources(scores, ATOMS, 10) == ["src_blue", "src_red"]
|
||||
|
||||
|
||||
def test_top_sources_rejects_a_length_mismatch() -> None:
|
||||
with pytest.raises(SystemExit, match="scores has 2 entries for 3 atoms"):
|
||||
top_sources(np.array([1.0, 2.0]), ATOMS, 3)
|
||||
|
||||
|
||||
def test_unit_scales_to_zero_one_and_flattens_a_constant() -> None:
|
||||
np.testing.assert_allclose(_unit(np.array([0.0, 2.0])), [0.0, 1.0])
|
||||
np.testing.assert_allclose(_unit(np.array([7.0, 7.0, 7.0])), [0.0, 0.0, 0.0])
|
||||
|
||||
|
||||
def test_fuse_uses_the_product_weights() -> None:
|
||||
fused = fuse(np.array([1.0, 0.0]), np.array([0.0, 1.0]), np.array([0.0, 0.0]))
|
||||
assert fused[0] == pytest.approx(0.55)
|
||||
assert fused[1] == pytest.approx(0.35)
|
||||
assert sum([0.55, 0.35, 0.10]) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_graph_only_arm_is_a_constant_and_sorts_by_source_id() -> None:
|
||||
runner = ArmRunner("graph_only", ATOMS, Path("/nonexistent"))
|
||||
assert runner.rank("key") == ["src_blue", "src_red"]
|
||||
|
||||
|
||||
def test_bm25_arm_finds_the_atom_that_names_the_query() -> None:
|
||||
runner = ArmRunner("bm25_only", ATOMS, Path("/nonexistent"))
|
||||
assert runner.rank("the key hangs")[0] == "src_red"
|
||||
# A query with no tokens has no signal: every score is zero, so the order is source id.
|
||||
assert runner.rank("") == ["src_blue", "src_red"]
|
||||
|
||||
|
||||
def test_arm_needs_reports_missing_models_without_crashing(tmp_path: Path) -> None:
|
||||
assert arm_needs("bm25_only", tmp_path) is True
|
||||
assert arm_needs("dense_only", tmp_path) is False
|
||||
(local_model_dir(tmp_path, "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")).mkdir(parents=True)
|
||||
assert arm_needs("dense_only", tmp_path) is True
|
||||
assert arm_needs("fusion_rerank", tmp_path) is False
|
||||
|
||||
|
||||
def test_arm_needs_rejects_an_unknown_arm() -> None:
|
||||
with pytest.raises(SystemExit, match="unknown arm"):
|
||||
arm_needs("quantum", Path("/nonexistent"))
|
||||
assert "fusion_rerank" in ARMS
|
||||
|
||||
|
||||
def test_local_model_dir_flattens_the_namespace() -> None:
|
||||
assert local_model_dir(Path("/m"), "BAAI/bge-reranker-base") == Path("/m/bge-reranker-base")
|
||||
|
||||
|
||||
def make_database(path: Path) -> None:
|
||||
"""Create the product's atoms table and put three rows in it."""
|
||||
connection = sqlite3.connect(path)
|
||||
connection.execute(
|
||||
"CREATE TABLE atoms (atom_id text PRIMARY KEY, source_id text, modality text, source text, time text,"
|
||||
" text text, image_ref text, embedding_refs_json text, layout_json text, relations_json text,"
|
||||
" trust_json text, token_equivalent integer, metadata_json text)"
|
||||
)
|
||||
for atom in ATOMS:
|
||||
connection.execute("INSERT INTO atoms (atom_id, source_id, text) VALUES (?, ?, ?)", (atom.atom_id, atom.source_id, atom.text))
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
|
||||
def test_load_atoms_returns_every_row_in_a_stable_order(tmp_path: Path) -> None:
|
||||
db = tmp_path / "ely_eye.sqlite"
|
||||
make_database(db)
|
||||
loaded = load_atoms(db)
|
||||
assert [atom.atom_id for atom in loaded] == ["a1", "a2", "a3"]
|
||||
assert loaded[2].text == "the key hangs on the red door"
|
||||
|
||||
|
||||
def test_load_atoms_fails_fast_on_a_missing_or_unusable_database(tmp_path: Path) -> None:
|
||||
with pytest.raises(SystemExit, match="no merged baseline database"):
|
||||
load_atoms(tmp_path / "absent.sqlite")
|
||||
no_table = tmp_path / "no_table.sqlite"
|
||||
no_table.touch()
|
||||
with pytest.raises(SystemExit, match="cannot read the atoms table"):
|
||||
load_atoms(no_table)
|
||||
empty = tmp_path / "empty.sqlite"
|
||||
make_database(empty)
|
||||
connection = sqlite3.connect(empty)
|
||||
connection.execute("DELETE FROM atoms")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
with pytest.raises(SystemExit, match="atoms table is empty"):
|
||||
load_atoms(empty)
|
||||
|
||||
|
||||
def test_percentile_needs_timings_and_a_valid_fraction() -> None:
|
||||
runner = ArmRunner("graph_only", ATOMS, Path("/nonexistent"))
|
||||
with pytest.raises(SystemExit, match="no timings recorded"):
|
||||
runner.percentile(0.5)
|
||||
runner.rank("key")
|
||||
with pytest.raises(SystemExit, match="percentile must be within"):
|
||||
runner.percentile(1.5)
|
||||
assert runner.percentile(0.5) >= 0.0
|
||||
Reference in new issue
Block a user