Stop two harnesses from measuring each other's store
A scratch copy is one run's working store. The video harness defaults to bench/runs/video_audio and copy_store() copies straight over it, so a smoke run started while the full run was mid-ingest replaced the database under it: the full run's after-arm would have scored a store whose speech atoms had been swapped out, and the smoke itself died on "database is locked" inside text_index. Six harnesses share that copy path, so the claim lives in bench/scratch.py: a live pid in held_by refuses the path, a stale one is taken over, and the same process may re-copy per arm. The reviewer's smoke knob also measured nothing instead of saying so. With --limit the probe sample narrows to video only, and a small --probes strides past that modality entirely, so the run crashed in aggregate() after the ingest. probe_sample() now switches the arm off at --probes 0 and fails in half a second otherwise. The frozen-report guard had two holes found by running it: it compared cwd-relative parents, so an absolute path to the frozen file passed, and it refused the harness's own default destination once that file existed, which would have stopped the shipped run. It now compares resolved paths and refuses only committed evidence. Record fixes from review round 3: the round-2 line credited a reproduction to the wrong round and quoted a git log -S hit for a string that never existed in code; the suite counts now name the commit each was measured on (401 at83dfc14, 406 ateab6bd7, 410 atcf2de31, 411 at92237c8). Gates: pytest -q 418 passed; ruff clean; bench.filesize_check OK; mypy cold cache and pyright clean on the four touched files.
This commit is contained in:
5 files changed
+171
-16
No files matched your search
@@ -0,0 +1,31 @@
|
||||
"""A scratch copy is one run's working store; two runs on one path silently measure each other's data."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from bench import scratch
|
||||
|
||||
|
||||
def test_a_live_holder_refuses_the_same_scratch(tmp_path: Path) -> None:
|
||||
"""The collision I hit: a second run re-copied the store under a run that was ingesting into it."""
|
||||
if not Path("/proc/1").exists():
|
||||
pytest.skip("liveness is read from /proc")
|
||||
(tmp_path / scratch.HELD).write_text("1", encoding="utf-8")
|
||||
with pytest.raises(SystemExit, match="is held by pid 1"):
|
||||
scratch.claim(tmp_path)
|
||||
|
||||
|
||||
def test_a_stale_claim_is_taken_over(tmp_path: Path) -> None:
|
||||
"""A crashed run must not lock the scratch away forever."""
|
||||
(tmp_path / scratch.HELD).write_text("999999", encoding="utf-8")
|
||||
scratch.claim(tmp_path)
|
||||
assert (tmp_path / scratch.HELD).read_text(encoding="utf-8") == str(os.getpid())
|
||||
|
||||
|
||||
def test_one_run_may_copy_more_than_once(tmp_path: Path) -> None:
|
||||
"""Harnesses that score several arms re-copy on purpose; that is not a collision."""
|
||||
scratch.claim(tmp_path)
|
||||
scratch.claim(tmp_path)
|
||||
assert (tmp_path / scratch.HELD).read_text(encoding="utf-8") == str(os.getpid())
|
||||
@@ -11,8 +11,9 @@ from bench import video_audio_check
|
||||
def test_a_frozen_report_is_not_overwritten_by_a_harness(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A reviewer runs harnesses in a worktree of its own; the committed reports stay put."""
|
||||
"""A reviewer runs harnesses in a worktree of its own; the committed evidence stays put."""
|
||||
monkeypatch.setattr(video_audio_check, "PAYLOAD", tmp_path / "video_audio.json")
|
||||
monkeypatch.setattr(video_audio_check, "_tracked", lambda path: True)
|
||||
frozen = tmp_path / "VIDEO_AUDIO.md"
|
||||
frozen.write_text("committed evidence", encoding="utf-8")
|
||||
with pytest.raises(SystemExit, match="is a frozen report"):
|
||||
@@ -23,6 +24,47 @@ def test_a_frozen_report_is_not_overwritten_by_a_harness(
|
||||
video_audio_check.refuse_frozen_overwrite(elsewhere)
|
||||
|
||||
|
||||
def test_a_harness_may_write_its_own_untracked_output(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The default destination is the reports directory; refusing it would stop the shipped run."""
|
||||
monkeypatch.setattr(video_audio_check, "PAYLOAD", tmp_path / "video_audio.json")
|
||||
monkeypatch.setattr(video_audio_check, "_tracked", lambda path: False)
|
||||
fresh = tmp_path / "VIDEO_AUDIO.md"
|
||||
fresh.write_text("not yet evidence", encoding="utf-8")
|
||||
video_audio_check.refuse_frozen_overwrite(fresh)
|
||||
|
||||
|
||||
def test_an_absolute_path_to_the_frozen_directory_is_refused_too(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The guard compares where the file really lands, not how the path was spelled."""
|
||||
monkeypatch.setattr(video_audio_check, "PAYLOAD", tmp_path / "video_audio.json")
|
||||
monkeypatch.setattr(video_audio_check, "_tracked", lambda path: True)
|
||||
frozen = tmp_path / "VIDEO_AUDIO.md"
|
||||
frozen.write_text("committed evidence", encoding="utf-8")
|
||||
with pytest.raises(SystemExit, match="is a frozen report"):
|
||||
video_audio_check.refuse_frozen_overwrite(Path(frozen.resolve()))
|
||||
|
||||
|
||||
def test_the_probe_arm_switches_off_rather_than_ranking_nothing() -> None:
|
||||
"""--probes 0 means "skip that arm"; a small count with the smoke knob means "you measured nothing"."""
|
||||
assert video_audio_check.probe_sample(0, 0) == []
|
||||
# The shipped sample keeps every modality; the smoke knob narrows it to video, where a
|
||||
# small count can miss the modality entirely (the reviewer's --limit 1 --probes 1 crash).
|
||||
assert any(query.modality == video_audio_check.VIDEO_MODALITY for query in video_audio_check.probe_sample(300, 0))
|
||||
for probes in (1, 5):
|
||||
with pytest.raises(SystemExit, match=f"--probes {probes} picked no answerable video query"):
|
||||
video_audio_check.probe_sample(probes, probes)
|
||||
|
||||
|
||||
def test_an_empty_video_arm_is_a_lie_not_a_zero() -> None:
|
||||
"""The video arm is the point of the harness; a row of zeros there must stop the run."""
|
||||
with pytest.raises(SystemExit, match="nothing to rank"):
|
||||
video_audio_check._required(None, "every video question")
|
||||
assert video_audio_check._score_of(None) == "skipped"
|
||||
|
||||
|
||||
def test_a_missing_speech_snapshot_says_so_before_ranking(tmp_path: Path) -> None:
|
||||
"""Without a snapshot every video defers, and the after row would equal the before row."""
|
||||
with pytest.raises(SystemExit, match="no speech snapshot"):
|
||||
|
||||
Reference in new issue
Block a user