ZacharyZhang-NY
f8a9428f5a
Round four of the independent review found seven record and coverage
defects. The functional side it verified by running; these are the fixes.
- `_tracked` asked the current checkout's index, so a reviewer in a worktree
pointing at the main checkout's committed report was told it was untracked.
It now asks the repository that owns the path, and the guard no longer
compares directories: any tracked file is committed evidence.
- `retrieval.py` claimed the dense index "catches up inside search". It does
not: `_stale` is only used by `sync()` and `stats()`, so a store upgraded
from v11 ranks an empty dense leg until `ely-eye build-dense-index` runs.
The docstring now states that, and `doctor` already names it.
- Four wirings had no discriminating test, each proven by mutation:
a stubbed `_tracked` (now a test uses the real git index), `copy_store`
claiming the scratch (now a live holder blocks the copy before a byte is
copied), `_stale` reading the atoms table (now a closed memory that loses
its vector is owed one), and the v10/v11 to v12 `DROP VIEW` gate (now a
hand-made v11 store is upgraded and its view body re-checked).
- The record corrected against the reviewer's measurements: the five
mutations did not each turn one test red (M1 turned two, M3 turned two,
`_stale` alone turned none), the pyright/mypy file set was wrong, and the
round added seven tests rather than three.
Gates: 422 passed; ruff clean; filesize clean; audit clean; mypy 24 errors in
9 files repo-wide, equal to the 9cb46c1 baseline; pyright 0 on the nine
touched files; zero whole-line comment blocks over 157 tracked files.
107 lines
5.0 KiB
Python
107 lines
5.0 KiB
Python
"""The video+audio harness must refuse to measure nothing, and must not touch frozen evidence."""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from bench import video_audio_check
|
|
|
|
|
|
def test_a_frozen_report_is_not_overwritten_by_a_harness(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""A reviewer runs harnesses in a worktree of its own; the committed evidence stays put, in any directory."""
|
|
monkeypatch.setattr(video_audio_check, "_tracked", lambda path: True)
|
|
frozen = tmp_path / "VIDEO_AUDIO.md"
|
|
frozen.write_text("committed evidence", encoding="utf-8")
|
|
with pytest.raises(SystemExit, match="is a frozen report"):
|
|
video_audio_check.refuse_frozen_overwrite(frozen)
|
|
elsewhere = tmp_path / "scratch" / "VIDEO_AUDIO.md"
|
|
elsewhere.parent.mkdir()
|
|
elsewhere.write_text("scratch", encoding="utf-8")
|
|
with pytest.raises(SystemExit, match="is a frozen report"):
|
|
video_audio_check.refuse_frozen_overwrite(elsewhere)
|
|
|
|
|
|
def test_a_harness_may_write_its_own_untracked_output(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""The default destination is the reports directory; refusing it would stop the shipped run."""
|
|
monkeypatch.setattr(video_audio_check, "_tracked", lambda path: False)
|
|
fresh = tmp_path / "VIDEO_AUDIO.md"
|
|
fresh.write_text("not yet evidence", encoding="utf-8")
|
|
video_audio_check.refuse_frozen_overwrite(fresh)
|
|
|
|
|
|
def test_an_absolute_path_to_the_frozen_directory_is_refused_too(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""The guard asks about the file, not the spelling of its path."""
|
|
monkeypatch.setattr(video_audio_check, "_tracked", lambda path: True)
|
|
frozen = tmp_path / "VIDEO_AUDIO.md"
|
|
frozen.write_text("committed evidence", encoding="utf-8")
|
|
with pytest.raises(SystemExit, match="is a frozen report"):
|
|
video_audio_check.refuse_frozen_overwrite(Path(frozen.resolve()))
|
|
|
|
|
|
def test_a_committed_report_is_frozen_without_any_stub(tmp_path: Path) -> None:
|
|
"""A stubbed `_tracked` proves the branch works, not that the guard does: this test uses the real git index."""
|
|
committed = Path("bench/reports/LEGS.md")
|
|
assert committed.exists(), "the acceptance needs a tracked report to refuse"
|
|
with pytest.raises(SystemExit, match="is a frozen report"):
|
|
video_audio_check.refuse_frozen_overwrite(committed)
|
|
elsewhere = tmp_path / "LEGS.md"
|
|
elsewhere.write_text("mine", encoding="utf-8")
|
|
video_audio_check.refuse_frozen_overwrite(elsewhere)
|
|
|
|
|
|
def test_the_probe_arm_switches_off_rather_than_ranking_nothing() -> None:
|
|
"""--probes 0 means "skip that arm"; a small count with the smoke knob means "you measured nothing"."""
|
|
assert video_audio_check.probe_sample(0, 0) == []
|
|
# The shipped sample keeps every modality; the smoke knob narrows it to video, where a small count can miss one entirely.
|
|
assert any(query.modality == video_audio_check.VIDEO_MODALITY for query in video_audio_check.probe_sample(300, 0))
|
|
for probes in (1, 5):
|
|
with pytest.raises(SystemExit, match=f"--probes {probes} picked no answerable video query"):
|
|
video_audio_check.probe_sample(probes, probes)
|
|
|
|
|
|
def test_an_empty_video_arm_is_a_lie_not_a_zero() -> None:
|
|
"""The video arm is the point of the harness; a row of zeros there must stop the run."""
|
|
with pytest.raises(SystemExit, match="nothing to rank"):
|
|
video_audio_check._required(None, "every video question")
|
|
assert video_audio_check._score_of(None) == "skipped"
|
|
|
|
|
|
def test_a_missing_speech_snapshot_says_so_before_ranking(tmp_path: Path) -> None:
|
|
"""Without a snapshot every video defers, and the after row would equal the before row."""
|
|
with pytest.raises(SystemExit, match="no speech snapshot"):
|
|
video_audio_check.add_speech(tmp_path, [tmp_path / "a.mp4"], tmp_path / "no-snapshot", 0)
|
|
|
|
|
|
def test_a_store_without_video_sources_refuses(tmp_path: Path) -> None:
|
|
"""Speech has to join to a video source; a store with none would measure a new corpus."""
|
|
import sqlite3
|
|
|
|
videos = tmp_path / "videomme"
|
|
videos.mkdir()
|
|
(videos / "a.mp4").write_bytes(b"\x00")
|
|
with pytest.raises(SystemExit, match="are not here"):
|
|
video_audio_check.guard_videos(tmp_path / "nowhere", tmp_path)
|
|
store = tmp_path / "store"
|
|
(store / "data").mkdir(parents=True)
|
|
connection = sqlite3.connect(store / "data" / "ely_eye.sqlite")
|
|
connection.execute("CREATE TABLE sources (source_id text, kind text)")
|
|
connection.commit()
|
|
connection.close()
|
|
with pytest.raises(SystemExit, match="holds no video sources"):
|
|
video_audio_check.guard_videos(videos, store)
|
|
|
|
|
|
def test_every_measured_video_query_carries_gold() -> None:
|
|
"""An answerable-only set: a query with no gold would score 0 against nothing."""
|
|
queries: list[Any] = video_audio_check.video_queries()
|
|
assert queries, "the frozen suite must carry answerable video queries"
|
|
assert all(query.modality == video_audio_check.VIDEO_MODALITY for query in queries)
|
|
assert all(query.gold for query in queries)
|