Make the speech numbers reproducible instead of merely printed

A review of 5289cb4 found the ASR report was not honest arithmetic:

- "audio s" was the last segment's end, not the media length. The real
  container durations are 88.93 / 109.18 / 60.07 s, so the report understated
  the audio by 48 s and every speed cell was wrong.
- Decoding was unpinned, so a rerun was a new draw: WER moved 0.242 to 0.262
  and one video moved 0.278 to 0.583. Greedy single-temperature decoding with
  no carry-over now answers the same way twice, and the harness runs the pass
  twice to show it.
- The report claimed its WER was an upper bound. It is not a bound in either
  direction: a caption that drops words also hides words the model dropped.
  The denominators (201 / 296 / 36 reference words) are printed instead.
- "66 speech atoms" was bench-side arithmetic. The store now says 80 audio
  atoms carrying 563 word timestamps, read back after a real ingest.

Same silent-skip bug in a second place: a text file over max_text_file_bytes
returned an empty batch with no reason. It raises FileTooLarge with its size,
and a walk records that refusal like any other.

Also fixed from the same review: the Go search tool never published the audio
modality, cli_senses carried a second json_dump, the no-audio-track branch had
a test that fed it markdown, _seconds was an empty wrapper, and the docstring
quoted a speed no report measured.

Gate root cause: bench/tests imports the bench package by name with no root
conftest, so bare pytest could not collect it at all. pythonpath in pyproject
fixes both launch styles; 212 tests pass either way.

Verification: 212 passed (bare pytest and python -m pytest), ruff clean,
mypy clean on the six touched files with a cold cache, repo-wide mypy 24/6
unchanged from HEAD, pyright 0 errors on the touched files, go build/vet/test
and gofmt clean.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 22:23:33 -04:00
1 parent 5289cb4b01
commit 709104e809
9 files changed
+306 -92

No files matched your search

+30 -9
View File
@@ -17,6 +17,14 @@ DOWNLOAD_HINT = (
DEFERRED = "deferred: no speech model snapshot"
MODEL_GRAPH = "model.bin"
SAMPLE_RATE = 16_000
CPU_THREADS = 4
TRANSCRIBE_KNOBS: dict[str, Any] = {
"word_timestamps": True,
"beam_size": 1,
"temperature": (0.0,),
"condition_on_previous_text": False,
"vad_filter": False,
}
@dataclass(frozen=True)
@@ -66,8 +74,18 @@ def decode_audio(path: Path) -> np.ndarray:
return np.concatenate(frames).astype(np.float32)
def _seconds(value: Any) -> float:
return float(value)
def audio_duration(path: Path) -> float:
"""Seconds of media the container claims; 0.0 when it does not say."""
import av
from av.error import FFmpegError
try:
with av.open(str(path)) as container:
if not container.duration:
return 0.0
return container.duration / av.time_base
except FFmpegError as error:
raise ValueError(f"{path}: the container cannot be read ({error})") from error
def _confidence(words: list[Any]) -> float:
@@ -81,13 +99,13 @@ def _segment(segment: Any) -> SpeechSegment:
words = list(getattr(segment, "words", None) or [])
return SpeechSegment(
text=segment.text.strip(),
start=_seconds(segment.start),
end=_seconds(segment.end),
start=float(segment.start),
end=float(segment.end),
words=tuple(
SpokenWord(
word=spoken.word.strip(),
start=_seconds(spoken.start),
end=_seconds(spoken.end),
start=float(spoken.start),
end=float(spoken.end),
confidence=float(getattr(spoken, "probability", 0.0) or 0.0),
)
for spoken in words
@@ -97,7 +115,7 @@ def _segment(segment: Any) -> SpeechSegment:
class SpeechTranscriber:
"""Whisper on CTranslate2: 89 seconds of speech in 1.5 seconds of CPU, int8."""
"""Whisper on CTranslate2: 88.9 seconds of speech in 1.7 seconds of CPU int8 (bench/reports/ASR.md)."""
def __init__(self, settings: Settings | None = None) -> None:
self.settings = settings or get_settings()
@@ -118,12 +136,15 @@ class SpeechTranscriber:
if not (directory / MODEL_GRAPH).exists():
raise SystemExit(f"{directory} holds no {MODEL_GRAPH}; {DOWNLOAD_HINT}")
model = self._load(directory)
segments, _info = model.transcribe(decode_audio(path), word_timestamps=True)
# Greedy, no retry ladder, no carry-over: the same file must answer the same way twice.
segments, _info = model.transcribe(decode_audio(path), **TRANSCRIBE_KNOBS)
return [_segment(segment) for segment in segments]
def _load(self, directory: Path) -> Any:
if self._model is None:
from faster_whisper import WhisperModel
self._model = WhisperModel(str(directory), device="cpu", compute_type="int8")
self._model = WhisperModel(
str(directory), device="cpu", compute_type="int8", cpu_threads=CPU_THREADS
)
return self._model
+1 -6
View File
@@ -9,12 +9,12 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import asdict
from pathlib import Path
from typing import Any
import typer
from .asr import DEFERRED as SPEECH_DEFERRED
from .asr import SpeechTranscriber
from .cli_common import json_dump
from .config import Settings
from .db import Database
from .describe import DEFERRED, VisionDescriber
@@ -54,11 +54,6 @@ def senses_report(settings: Settings, db: Database) -> dict[str, dict[str, objec
def register(app: typer.Typer, settings: Callable[[], Settings]) -> None:
"""Add the media commands. The settings thunk lets one patch in a test redirect every command."""
def json_dump(value: Any) -> str:
import json
return json.dumps(value, ensure_ascii=False, indent=2)
@app.command()
def transcribe(path: Path) -> None:
"""Words with the instant each was spoken; the snapshot stays the operator's to supply."""
+6 -3
View File
@@ -16,6 +16,7 @@ from .ingestion_common import (
SUPPORTED_IMAGE_EXTENSIONS,
SUPPORTED_TEXT_EXTENSIONS,
SUPPORTED_VIDEO_EXTENSIONS,
FileTooLarge,
IngestBatch,
UnsupportedFile,
)
@@ -62,7 +63,7 @@ class IngestionService(MediaIngest, RepositoryIngest):
for file_path in files:
try:
file_batch = self._ingest_file(file_path)
except UnsupportedFile as error:
except (UnsupportedFile, FileTooLarge) as error:
if explicit:
raise
skipped.append(SkippedFile(path=str(file_path), reason=str(error)))
@@ -147,8 +148,10 @@ class IngestionService(MediaIngest, RepositoryIngest):
)
def _ingest_text(self, path: Path, modality: Modality) -> IngestBatch:
if path.stat().st_size > self.settings.max_text_file_bytes:
return IngestBatch(sources=[], atoms=[])
limit = self.settings.max_text_file_bytes
size = path.stat().st_size
if size > limit:
raise FileTooLarge(f"{path} is {size} bytes, over max_text_file_bytes={limit}")
source = self._source_record(path, modality.value)
text = path.read_text(encoding="utf-8", errors="replace")
atom = EvidenceAtom(
+4
View File
@@ -54,6 +54,10 @@ class UnsupportedFile(ValueError):
"""No reader for this file: a walk records the reason, an explicit path fails."""
class FileTooLarge(ValueError):
"""A reader exists but the file is over the configured cap: refused with its size stated."""
@dataclass
class IngestBatch:
sources: list[SourceRecord]