Let the harness tests run on a machine that has no models

CI failed four tests with "SystemExit: bench/models/paraphrase-multilingual-
MiniLM-L12-v2: model not downloaded". The claim those tests make is that the
joint ablation pass ranks exactly like the per-arm pass and that three arms
needing BM25 score the corpus once. That is a claim about the harness, not
about embedding quality, so it should not require a 470MB download to check.

The tests now inject a deterministic 384-dimensional stand-in dense model and
a stand-in cross-encoder at the seam both paths share. The real models still
carry every measurement in bench/reports; nothing measured changed.

Proven by the condition that broke: with both model directories moved out of
the tree, the whole suite passes (253 passed).
This commit is contained in:
ZacharyZhang-NY committed 2026-10-08 03:04:45 -04:00
1 parent 07825913fe
commit 4b367cb59b
1 file changed
+40
+40
View File
@@ -6,10 +6,13 @@ drifted from the per-arm path, the baseline would silently measure a different s
from __future__ import annotations from __future__ import annotations
import hashlib
import json import json
from collections.abc import Sequence
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
import numpy as np
import pytest import pytest
from bench import systems from bench import systems
@@ -33,9 +36,45 @@ def _queries() -> list[str]:
return ["alpha report", "beta invoice", "gamma", ""] return ["alpha report", "beta invoice", "gamma", ""]
class _TinyDense:
"""Stand-in embedding model: these tests ask whether two harness paths agree, not
whether embeddings are good, so they must run without a 470MB download."""
def encode(self, texts: Sequence[str], **_: Any) -> np.ndarray:
return np.stack([_unit_vector(str(text)) for text in texts]).astype(np.float32)
class _TinyReranker:
"""Stand-in cross-encoder with the same scores(query, texts) shape as Reranker."""
def __init__(self, model_dir: Path) -> None:
self.model_dir = model_dir
def scores(self, query: str, texts: Sequence[str]) -> np.ndarray:
return np.array([_score(query, text) for text in texts], dtype=np.float64)
def _unit_vector(text: str) -> np.ndarray:
seed = int(hashlib.sha256(text.encode()).hexdigest()[:8], 16)
vector = np.random.default_rng(seed).standard_normal(systems.DENSE_DIM)
return vector / np.linalg.norm(vector)
def _score(query: str, text: str) -> float:
digest = hashlib.sha256(f"{query}\n{text}".encode()).hexdigest()
return int(digest[:8], 16) / 0xFFFFFFFF
def _stub_models(monkeypatch: pytest.MonkeyPatch) -> None:
dense_dir = systems.local_model_dir(systems.MODELS, systems.DENSE_MODEL)
monkeypatch.setattr(systems, "_MODELS", {str(dense_dir): _TinyDense()})
monkeypatch.setattr(systems, "Reranker", _TinyReranker)
@pytest.mark.parametrize("arm", MODEL_ARMS) @pytest.mark.parametrize("arm", MODEL_ARMS)
def test_joint_ranking_matches_the_per_arm_runner(arm: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_joint_ranking_matches_the_per_arm_runner(arm: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(systems, "INDEX_DIR", tmp_path) monkeypatch.setattr(systems, "INDEX_DIR", tmp_path)
_stub_models(monkeypatch)
queries = _queries() queries = _queries()
joint = JointRunner([arm], ATOMS, systems.MODELS) joint = JointRunner([arm], ATOMS, systems.MODELS)
per_arm = ArmRunner(arm, ATOMS, systems.MODELS) per_arm = ArmRunner(arm, ATOMS, systems.MODELS)
@@ -50,6 +89,7 @@ def test_joint_runner_computes_the_sparse_signal_once_per_query(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None: ) -> None:
monkeypatch.setattr(systems, "INDEX_DIR", tmp_path) monkeypatch.setattr(systems, "INDEX_DIR", tmp_path)
_stub_models(monkeypatch)
joint = JointRunner(["bm25_only", "no_rerank", "fusion_rerank"], ATOMS, systems.MODELS) joint = JointRunner(["bm25_only", "no_rerank", "fusion_rerank"], ATOMS, systems.MODELS)
calls = 0 calls = 0
assert joint._sparse is not None assert joint._sparse is not None