ZacharyZhang-NY
cf2de313dc
Round 2 found the functional side true and six record/discipline defects. - D9: the write-path line said "comparison uses secrets.compare_digest" as if it covered both tokens. It does not: the operator's plaintext is compared with compare_digest (api_auth.py:54, :110), a holder token is SQL equality on its sha256 digest (holder_store.verify_token) because no plaintext is stored. The boundary is now stated in verify_token's docstring as well as the record. - D10/D13: three historical numbers were written as if they were current. The four-file 59 is pinned to2ea48d8(test_trust_quarantine.py was 25 tests there, 26 fromd56996bon), and both 401 gate lines are pinned to the commit they were measured on (83dfc14,89c880f) with the current 406 named next to them. - D12: the compiler cache-key line overstated one direction. The old key had no identity part at all, so whoever compiled first got served to the next caller; an empty pack reaching the operator is one direction, not the mechanism. - D11: the p2-5 acceptance cell's suite count is pinned the same way. - D14: bench/video_audio_check.py wrote straight into bench/reports/, so a reviewer running it in its own worktree would have overwritten frozen evidence. It now takes --report/--payload and refuses a destination that already holds a frozen report; refuse_frozen_overwrite is tested, along with the guards that stop the harness from measuring nothing. Gates: pytest -q 410 passed; ruff clean; bench.filesize_check OK; bench.audit --product 33 harness files / 2 reports / 6 set files, clean; pyright 0 errors on the three touched files; mypy cold cache Success on the same three.
57 lines
2.4 KiB
Python
57 lines
2.4 KiB
Python
"""The video+audio harness must refuse to measure nothing, and must not touch frozen evidence."""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from bench import video_audio_check
|
|
|
|
|
|
def test_a_frozen_report_is_not_overwritten_by_a_harness(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""A reviewer runs harnesses in a worktree of its own; the committed reports stay put."""
|
|
monkeypatch.setattr(video_audio_check, "PAYLOAD", tmp_path / "video_audio.json")
|
|
frozen = tmp_path / "VIDEO_AUDIO.md"
|
|
frozen.write_text("committed evidence", encoding="utf-8")
|
|
with pytest.raises(SystemExit, match="is a frozen report"):
|
|
video_audio_check.refuse_frozen_overwrite(frozen)
|
|
elsewhere = tmp_path / "scratch" / "VIDEO_AUDIO.md"
|
|
elsewhere.parent.mkdir()
|
|
elsewhere.write_text("scratch", encoding="utf-8")
|
|
video_audio_check.refuse_frozen_overwrite(elsewhere)
|
|
|
|
|
|
def test_a_missing_speech_snapshot_says_so_before_ranking(tmp_path: Path) -> None:
|
|
"""Without a snapshot every video defers, and the after row would equal the before row."""
|
|
with pytest.raises(SystemExit, match="no speech snapshot"):
|
|
video_audio_check.add_speech(tmp_path, [tmp_path / "a.mp4"], tmp_path / "no-snapshot", 0)
|
|
|
|
|
|
def test_a_store_without_video_sources_refuses(tmp_path: Path) -> None:
|
|
"""Speech has to join to a video source; a store with none would measure a new corpus."""
|
|
import sqlite3
|
|
|
|
videos = tmp_path / "videomme"
|
|
videos.mkdir()
|
|
(videos / "a.mp4").write_bytes(b"\x00")
|
|
with pytest.raises(SystemExit, match="are not here"):
|
|
video_audio_check.guard_videos(tmp_path / "nowhere", tmp_path)
|
|
store = tmp_path / "store"
|
|
(store / "data").mkdir(parents=True)
|
|
connection = sqlite3.connect(store / "data" / "ely_eye.sqlite")
|
|
connection.execute("CREATE TABLE sources (source_id text, kind text)")
|
|
connection.commit()
|
|
connection.close()
|
|
with pytest.raises(SystemExit, match="holds no video sources"):
|
|
video_audio_check.guard_videos(videos, store)
|
|
|
|
|
|
def test_every_measured_video_query_carries_gold() -> None:
|
|
"""An answerable-only set: a query with no gold would score 0 against nothing."""
|
|
queries: list[Any] = video_audio_check.video_queries()
|
|
assert queries, "the frozen suite must carry answerable video queries"
|
|
assert all(query.modality == video_audio_check.VIDEO_MODALITY for query in queries)
|
|
assert all(query.gold for query in queries)
|