Let the harness tests run on a machine that has no models
CI failed four tests with "SystemExit: bench/models/paraphrase-multilingual- MiniLM-L12-v2: model not downloaded". The claim those tests make is that the joint ablation pass ranks exactly like the per-arm pass and that three arms needing BM25 score the corpus once. That is a claim about the harness, not about embedding quality, so it should not require a 470MB download to check. The tests now inject a deterministic 384-dimensional stand-in dense model and a stand-in cross-encoder at the seam both paths share. The real models still carry every measurement in bench/reports; nothing measured changed. Proven by the condition that broke: with both model directories moved out of the tree, the whole suite passes (253 passed).
This commit is contained in:
1 parent
07825913fe
commit
4b367cb59b
1 file changed
+40
@@ -6,10 +6,13 @@ drifted from the per-arm path, the baseline would silently measure a different s
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
from collections.abc import Sequence
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from bench import systems
|
from bench import systems
|
||||||
@@ -33,9 +36,45 @@ def _queries() -> list[str]:
|
|||||||
return ["alpha report", "beta invoice", "gamma", ""]
|
return ["alpha report", "beta invoice", "gamma", ""]
|
||||||
|
|
||||||
|
|
||||||
|
class _TinyDense:
|
||||||
|
"""Stand-in embedding model: these tests ask whether two harness paths agree, not
|
||||||
|
whether embeddings are good, so they must run without a 470MB download."""
|
||||||
|
|
||||||
|
def encode(self, texts: Sequence[str], **_: Any) -> np.ndarray:
|
||||||
|
return np.stack([_unit_vector(str(text)) for text in texts]).astype(np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
class _TinyReranker:
|
||||||
|
"""Stand-in cross-encoder with the same scores(query, texts) shape as Reranker."""
|
||||||
|
|
||||||
|
def __init__(self, model_dir: Path) -> None:
|
||||||
|
self.model_dir = model_dir
|
||||||
|
|
||||||
|
def scores(self, query: str, texts: Sequence[str]) -> np.ndarray:
|
||||||
|
return np.array([_score(query, text) for text in texts], dtype=np.float64)
|
||||||
|
|
||||||
|
|
||||||
|
def _unit_vector(text: str) -> np.ndarray:
|
||||||
|
seed = int(hashlib.sha256(text.encode()).hexdigest()[:8], 16)
|
||||||
|
vector = np.random.default_rng(seed).standard_normal(systems.DENSE_DIM)
|
||||||
|
return vector / np.linalg.norm(vector)
|
||||||
|
|
||||||
|
|
||||||
|
def _score(query: str, text: str) -> float:
|
||||||
|
digest = hashlib.sha256(f"{query}\n{text}".encode()).hexdigest()
|
||||||
|
return int(digest[:8], 16) / 0xFFFFFFFF
|
||||||
|
|
||||||
|
|
||||||
|
def _stub_models(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
dense_dir = systems.local_model_dir(systems.MODELS, systems.DENSE_MODEL)
|
||||||
|
monkeypatch.setattr(systems, "_MODELS", {str(dense_dir): _TinyDense()})
|
||||||
|
monkeypatch.setattr(systems, "Reranker", _TinyReranker)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("arm", MODEL_ARMS)
|
@pytest.mark.parametrize("arm", MODEL_ARMS)
|
||||||
def test_joint_ranking_matches_the_per_arm_runner(arm: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_joint_ranking_matches_the_per_arm_runner(arm: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
monkeypatch.setattr(systems, "INDEX_DIR", tmp_path)
|
monkeypatch.setattr(systems, "INDEX_DIR", tmp_path)
|
||||||
|
_stub_models(monkeypatch)
|
||||||
queries = _queries()
|
queries = _queries()
|
||||||
joint = JointRunner([arm], ATOMS, systems.MODELS)
|
joint = JointRunner([arm], ATOMS, systems.MODELS)
|
||||||
per_arm = ArmRunner(arm, ATOMS, systems.MODELS)
|
per_arm = ArmRunner(arm, ATOMS, systems.MODELS)
|
||||||
@@ -50,6 +89,7 @@ def test_joint_runner_computes_the_sparse_signal_once_per_query(
|
|||||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
) -> None:
|
) -> None:
|
||||||
monkeypatch.setattr(systems, "INDEX_DIR", tmp_path)
|
monkeypatch.setattr(systems, "INDEX_DIR", tmp_path)
|
||||||
|
_stub_models(monkeypatch)
|
||||||
joint = JointRunner(["bm25_only", "no_rerank", "fusion_rerank"], ATOMS, systems.MODELS)
|
joint = JointRunner(["bm25_only", "no_rerank", "fusion_rerank"], ATOMS, systems.MODELS)
|
||||||
calls = 0
|
calls = 0
|
||||||
assert joint._sparse is not None
|
assert joint._sparse is not None
|
||||||
|
|||||||
Reference in new issue
Block a user