Make the speech numbers reproducible instead of merely printed
A review of 5289cb4 found the ASR report was not honest arithmetic:
- "audio s" was the last segment's end, not the media length. The real
container durations are 88.93 / 109.18 / 60.07 s, so the report understated
the audio by 48 s and every speed cell was wrong.
- Decoding was unpinned, so a rerun was a new draw: WER moved 0.242 to 0.262
and one video moved 0.278 to 0.583. Greedy single-temperature decoding with
no carry-over now answers the same way twice, and the harness runs the pass
twice to show it.
- The report claimed its WER was an upper bound. It is not a bound in either
direction: a caption that drops words also hides words the model dropped.
The denominators (201 / 296 / 36 reference words) are printed instead.
- "66 speech atoms" was bench-side arithmetic. The store now says 80 audio
atoms carrying 563 word timestamps, read back after a real ingest.
Same silent-skip bug in a second place: a text file over max_text_file_bytes
returned an empty batch with no reason. It raises FileTooLarge with its size,
and a walk records that refusal like any other.
Also fixed from the same review: the Go search tool never published the audio
modality, cli_senses carried a second json_dump, the no-audio-track branch had
a test that fed it markdown, _seconds was an empty wrapper, and the docstring
quoted a speed no report measured.
Gate root cause: bench/tests imports the bench package by name with no root
conftest, so bare pytest could not collect it at all. pythonpath in pyproject
fixes both launch styles; 212 tests pass either way.
Verification: 212 passed (bare pytest and python -m pytest), ruff clean,
mypy clean on the six touched files with a cold cache, repo-wide mypy 24/6
unchanged from HEAD, pyright 0 errors on the touched files, go build/vet/test
and gofmt clean.
This commit is contained in:
1 parent
5289cb4b01
commit
709104e809
9 files changed
+306
-92
No files matched your search
@@ -17,6 +17,14 @@ DOWNLOAD_HINT = (
|
||||
DEFERRED = "deferred: no speech model snapshot"
|
||||
MODEL_GRAPH = "model.bin"
|
||||
SAMPLE_RATE = 16_000
|
||||
CPU_THREADS = 4
|
||||
TRANSCRIBE_KNOBS: dict[str, Any] = {
|
||||
"word_timestamps": True,
|
||||
"beam_size": 1,
|
||||
"temperature": (0.0,),
|
||||
"condition_on_previous_text": False,
|
||||
"vad_filter": False,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -66,8 +74,18 @@ def decode_audio(path: Path) -> np.ndarray:
|
||||
return np.concatenate(frames).astype(np.float32)
|
||||
|
||||
|
||||
def _seconds(value: Any) -> float:
|
||||
return float(value)
|
||||
def audio_duration(path: Path) -> float:
|
||||
"""Seconds of media the container claims; 0.0 when it does not say."""
|
||||
import av
|
||||
from av.error import FFmpegError
|
||||
|
||||
try:
|
||||
with av.open(str(path)) as container:
|
||||
if not container.duration:
|
||||
return 0.0
|
||||
return container.duration / av.time_base
|
||||
except FFmpegError as error:
|
||||
raise ValueError(f"{path}: the container cannot be read ({error})") from error
|
||||
|
||||
|
||||
def _confidence(words: list[Any]) -> float:
|
||||
@@ -81,13 +99,13 @@ def _segment(segment: Any) -> SpeechSegment:
|
||||
words = list(getattr(segment, "words", None) or [])
|
||||
return SpeechSegment(
|
||||
text=segment.text.strip(),
|
||||
start=_seconds(segment.start),
|
||||
end=_seconds(segment.end),
|
||||
start=float(segment.start),
|
||||
end=float(segment.end),
|
||||
words=tuple(
|
||||
SpokenWord(
|
||||
word=spoken.word.strip(),
|
||||
start=_seconds(spoken.start),
|
||||
end=_seconds(spoken.end),
|
||||
start=float(spoken.start),
|
||||
end=float(spoken.end),
|
||||
confidence=float(getattr(spoken, "probability", 0.0) or 0.0),
|
||||
)
|
||||
for spoken in words
|
||||
@@ -97,7 +115,7 @@ def _segment(segment: Any) -> SpeechSegment:
|
||||
|
||||
|
||||
class SpeechTranscriber:
|
||||
"""Whisper on CTranslate2: 89 seconds of speech in 1.5 seconds of CPU, int8."""
|
||||
"""Whisper on CTranslate2: 88.9 seconds of speech in 1.7 seconds of CPU int8 (bench/reports/ASR.md)."""
|
||||
|
||||
def __init__(self, settings: Settings | None = None) -> None:
|
||||
self.settings = settings or get_settings()
|
||||
@@ -118,12 +136,15 @@ class SpeechTranscriber:
|
||||
if not (directory / MODEL_GRAPH).exists():
|
||||
raise SystemExit(f"{directory} holds no {MODEL_GRAPH}; {DOWNLOAD_HINT}")
|
||||
model = self._load(directory)
|
||||
segments, _info = model.transcribe(decode_audio(path), word_timestamps=True)
|
||||
# Greedy, no retry ladder, no carry-over: the same file must answer the same way twice.
|
||||
segments, _info = model.transcribe(decode_audio(path), **TRANSCRIBE_KNOBS)
|
||||
return [_segment(segment) for segment in segments]
|
||||
|
||||
def _load(self, directory: Path) -> Any:
|
||||
if self._model is None:
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
self._model = WhisperModel(str(directory), device="cpu", compute_type="int8")
|
||||
self._model = WhisperModel(
|
||||
str(directory), device="cpu", compute_type="int8", cpu_threads=CPU_THREADS
|
||||
)
|
||||
return self._model
|
||||
@@ -9,12 +9,12 @@ from __future__ import annotations
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import typer
|
||||
|
||||
from .asr import DEFERRED as SPEECH_DEFERRED
|
||||
from .asr import SpeechTranscriber
|
||||
from .cli_common import json_dump
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .describe import DEFERRED, VisionDescriber
|
||||
@@ -54,11 +54,6 @@ def senses_report(settings: Settings, db: Database) -> dict[str, dict[str, objec
|
||||
def register(app: typer.Typer, settings: Callable[[], Settings]) -> None:
|
||||
"""Add the media commands. The settings thunk lets one patch in a test redirect every command."""
|
||||
|
||||
def json_dump(value: Any) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
@app.command()
|
||||
def transcribe(path: Path) -> None:
|
||||
"""Words with the instant each was spoken; the snapshot stays the operator's to supply."""
|
||||
|
||||
@@ -16,6 +16,7 @@ from .ingestion_common import (
|
||||
SUPPORTED_IMAGE_EXTENSIONS,
|
||||
SUPPORTED_TEXT_EXTENSIONS,
|
||||
SUPPORTED_VIDEO_EXTENSIONS,
|
||||
FileTooLarge,
|
||||
IngestBatch,
|
||||
UnsupportedFile,
|
||||
)
|
||||
@@ -62,7 +63,7 @@ class IngestionService(MediaIngest, RepositoryIngest):
|
||||
for file_path in files:
|
||||
try:
|
||||
file_batch = self._ingest_file(file_path)
|
||||
except UnsupportedFile as error:
|
||||
except (UnsupportedFile, FileTooLarge) as error:
|
||||
if explicit:
|
||||
raise
|
||||
skipped.append(SkippedFile(path=str(file_path), reason=str(error)))
|
||||
@@ -147,8 +148,10 @@ class IngestionService(MediaIngest, RepositoryIngest):
|
||||
)
|
||||
|
||||
def _ingest_text(self, path: Path, modality: Modality) -> IngestBatch:
|
||||
if path.stat().st_size > self.settings.max_text_file_bytes:
|
||||
return IngestBatch(sources=[], atoms=[])
|
||||
limit = self.settings.max_text_file_bytes
|
||||
size = path.stat().st_size
|
||||
if size > limit:
|
||||
raise FileTooLarge(f"{path} is {size} bytes, over max_text_file_bytes={limit}")
|
||||
source = self._source_record(path, modality.value)
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
atom = EvidenceAtom(
|
||||
|
||||
@@ -54,6 +54,10 @@ class UnsupportedFile(ValueError):
|
||||
"""No reader for this file: a walk records the reason, an explicit path fails."""
|
||||
|
||||
|
||||
class FileTooLarge(ValueError):
|
||||
"""A reader exists but the file is over the configured cap: refused with its size stated."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestBatch:
|
||||
sources: list[SourceRecord]
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -9,10 +10,19 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from ely_eye import ingestion_media as media_module
|
||||
from ely_eye.asr import DEFERRED, SpeechSegment, SpeechTranscriber, SpokenWord, decode_audio
|
||||
from ely_eye.asr import (
|
||||
DEFERRED,
|
||||
SAMPLE_RATE,
|
||||
SpeechSegment,
|
||||
SpeechTranscriber,
|
||||
SpokenWord,
|
||||
audio_duration,
|
||||
decode_audio,
|
||||
)
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.ingestion import IngestionService, UnsupportedFile
|
||||
from ely_eye.ingestion_common import FileTooLarge
|
||||
from ely_eye.ocr import OcrResult
|
||||
from ely_eye.schemas import Modality
|
||||
|
||||
@@ -89,12 +99,18 @@ class FakeCv2:
|
||||
return True, FakeEncoded()
|
||||
|
||||
|
||||
def _settings(tmp_path: Path, asr_model_dir: Path | None = None) -> Settings:
|
||||
return Settings(home=tmp_path / "home", asr_model_dir=asr_model_dir)
|
||||
def _settings(tmp_path: Path, asr_model_dir: Path | None = None, max_text_file_bytes: int = 10_000_000) -> Settings:
|
||||
return Settings(home=tmp_path / "home", asr_model_dir=asr_model_dir, max_text_file_bytes=max_text_file_bytes)
|
||||
|
||||
|
||||
def _service(tmp_path: Path, transcriber: Any) -> tuple[IngestionService, Database]:
|
||||
settings = _settings(tmp_path)
|
||||
def _service(
|
||||
tmp_path: Path,
|
||||
transcriber: Any,
|
||||
*,
|
||||
asr_model_dir: Path | None = None,
|
||||
max_text_file_bytes: int = 10_000_000,
|
||||
) -> tuple[IngestionService, Database]:
|
||||
settings = _settings(tmp_path, asr_model_dir=asr_model_dir, max_text_file_bytes=max_text_file_bytes)
|
||||
settings.ensure_dirs()
|
||||
database = Database(settings)
|
||||
service = IngestionService(settings=settings, database=database)
|
||||
@@ -202,7 +218,71 @@ def test_the_transcriber_names_the_snapshot_it_wants(tmp_path: Path) -> None:
|
||||
half.transcribe(_audio(tmp_path))
|
||||
|
||||
|
||||
def _video_only(path: Path) -> Path:
|
||||
"""A real mp4 with a video stream and no audio track, written by the decoder we already ship."""
|
||||
import av
|
||||
|
||||
with av.open(str(path), "w") as container:
|
||||
stream = container.add_stream("mpeg4", rate=10)
|
||||
stream.width, stream.height, stream.pix_fmt = 64, 64, "yuv420p"
|
||||
for _ in range(5):
|
||||
frame = av.VideoFrame.from_ndarray(np.zeros((64, 64, 3), dtype=np.uint8), format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
return path
|
||||
|
||||
|
||||
def test_a_file_with_no_audio_track_says_so(tmp_path: Path) -> None:
|
||||
video = _video_only(tmp_path / "silent.mp4")
|
||||
|
||||
with pytest.raises(ValueError, match="the file holds no audio track"):
|
||||
decode_audio(video)
|
||||
|
||||
|
||||
def test_an_audio_track_with_no_frames_says_so(tmp_path: Path) -> None:
|
||||
silent = tmp_path / "silent.wav"
|
||||
with wave.open(str(silent), "wb") as writer:
|
||||
writer.setnchannels(1)
|
||||
writer.setsampwidth(2)
|
||||
writer.setframerate(SAMPLE_RATE)
|
||||
|
||||
with pytest.raises(ValueError, match="the audio track is empty"):
|
||||
decode_audio(silent)
|
||||
|
||||
|
||||
def test_a_walk_records_a_file_over_the_size_cap(tmp_path: Path) -> None:
|
||||
service, database = _service(tmp_path, FakeTranscriber(available=False), max_text_file_bytes=32)
|
||||
folder = tmp_path / "shelf"
|
||||
folder.mkdir()
|
||||
(folder / "note.md").write_text("# 钥匙放在抽屉里\n", encoding="utf-8")
|
||||
(folder / "huge.json").write_text('{"钥匙": "抽屉"}' * 4, encoding="utf-8")
|
||||
|
||||
response = service.ingest_path(folder)
|
||||
|
||||
assert response.atom_count > 0, "the small file is still ingested"
|
||||
assert [item.path for item in response.skipped] == [str(folder / "huge.json")]
|
||||
assert "over max_text_file_bytes=32" in response.skipped[0].reason
|
||||
assert database.atom_count() == response.atom_count
|
||||
|
||||
|
||||
def test_an_explicit_file_over_the_size_cap_fails_loudly(tmp_path: Path) -> None:
|
||||
service, _database = _service(tmp_path, FakeTranscriber(available=False), max_text_file_bytes=16)
|
||||
huge = tmp_path / "huge.json"
|
||||
huge.write_text('{"钥匙": "抽屉"}' * 4, encoding="utf-8")
|
||||
|
||||
with pytest.raises(FileTooLarge, match="over max_text_file_bytes=16"):
|
||||
service.ingest_path(huge)
|
||||
|
||||
|
||||
def test_the_container_duration_is_the_media_length(tmp_path: Path) -> None:
|
||||
video = _video_only(tmp_path / "silent.mp4")
|
||||
|
||||
assert audio_duration(video) == pytest.approx(0.5, abs=0.05)
|
||||
|
||||
|
||||
def test_a_text_file_cannot_be_decoded(tmp_path: Path) -> None:
|
||||
markdown = tmp_path / "notes.md"
|
||||
markdown.write_text("# 钥匙\n", encoding="utf-8")
|
||||
|
||||
|
||||
Reference in new issue
Block a user