Give spoken words a memory with an instant attached
An audio file used to produce no memory at all: _ingest_file returned an empty batch for any suffix it did not know, with no log and no reason. It now raises UnsupportedFile when you name the file and records a SkippedFile when it was a directory walk, so a refusal is visible either way. Speech is measured, not promised. Over 210.5 seconds of real Video-MME audio, faster-whisper-tiny on CPU int8 transcribed at 51x realtime and reached word error rate 0.242 against community subtitles nobody here wrote (a caption track drops words a transcript keeps, so that is an upper bound). 66 speech atoms now carry 550 word-level timestamps, 0 words out of order and 0 starting before their own segment. The snapshot is 78.2 MB, under the 500 MB cap, and the report is bench/reports/ASR.md. Two library facts were measured rather than assumed. faster-whisper's own audio loader is broken against PyAV 19 (av.open got an unexpected keyword 'metadata_errors') and it declares av with no version bound, so asr.decode_audio decodes with PyAV directly and hands the model a float32 array instead of pinning an old av with no cp314 wheel. And av.Error does not exist on PyAV 19 - the real base is av.error.FFmpegError - so that is the only thing caught and my own ValueErrors keep their messages. Files over the 500-line rule were split rather than excused: ingestion.py (748) into ingestion_common / ingestion_media / ingestion_repository / ingestion, and cli.py (544) into cli_senses plus cli_common, following the cli_transactive register pattern. MediaIngest declares _source_record abstract, so a half-wired host fails at construction instead of returning None. Verification: 207 tests pass; ruff clean on the 13 touched files; mypy clean on them, and repo-wide mypy fell from 29 errors to 24 (5 fixed, none added); pyright reports 0 errors on the touched files.
This commit is contained in:
1 parent
03ca35763f
commit
5289cb4b01
20 files changed
+1513
-611
No files matched your search
@@ -0,0 +1,129 @@
|
||||
"""Speech to text with word-level timings, so a spoken memory can be found and pointed at."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .config import Settings, get_settings
|
||||
|
||||
DOWNLOAD_HINT = (
|
||||
"fetch a faster-whisper CTranslate2 snapshot and set ELY_EYE_ASR_MODEL_DIR "
|
||||
"to the directory holding it"
|
||||
)
|
||||
DEFERRED = "deferred: no speech model snapshot"
|
||||
MODEL_GRAPH = "model.bin"
|
||||
SAMPLE_RATE = 16_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpokenWord:
|
||||
word: str
|
||||
start: float
|
||||
end: float
|
||||
confidence: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SpeechSegment:
|
||||
text: str
|
||||
start: float
|
||||
end: float
|
||||
words: tuple[SpokenWord, ...]
|
||||
confidence: float
|
||||
|
||||
|
||||
def decode_audio(path: Path) -> np.ndarray:
|
||||
"""Decode any container PyAV reads into mono 16 kHz floats, the shape the model wants.
|
||||
|
||||
The decode lives here rather than inside faster-whisper because its own decoder calls
|
||||
av.open(metadata_errors=...), a keyword PyAV 19 no longer accepts. PyAV 19 raises its
|
||||
failures as OSError and ValueError subclasses, so FFmpegError is the only thing caught
|
||||
here and a caller's own error keeps its own message.
|
||||
"""
|
||||
import av
|
||||
from av.error import FFmpegError
|
||||
|
||||
frames: list[np.ndarray] = []
|
||||
try:
|
||||
with av.open(str(path)) as container:
|
||||
audio_streams = container.streams.audio
|
||||
if not audio_streams:
|
||||
raise ValueError(f"{path}: the file holds no audio track")
|
||||
resampler = av.AudioResampler(format="flt", layout="mono", rate=SAMPLE_RATE)
|
||||
for chunk in container.decode(audio_streams[0]):
|
||||
if not isinstance(chunk, av.AudioFrame):
|
||||
continue
|
||||
for resampled in resampler.resample(chunk):
|
||||
frames.append(np.asarray(resampled.to_ndarray()).reshape(-1))
|
||||
except FFmpegError as error:
|
||||
raise ValueError(f"{path}: the audio cannot be decoded ({error})") from error
|
||||
if not frames:
|
||||
raise ValueError(f"{path}: the audio track is empty")
|
||||
return np.concatenate(frames).astype(np.float32)
|
||||
|
||||
|
||||
def _seconds(value: Any) -> float:
|
||||
return float(value)
|
||||
|
||||
|
||||
def _confidence(words: list[Any]) -> float:
|
||||
scores = [float(word.probability) for word in words if getattr(word, "probability", None) is not None]
|
||||
if not scores:
|
||||
return 0.0
|
||||
return min(sum(scores) / len(scores), 1.0)
|
||||
|
||||
|
||||
def _segment(segment: Any) -> SpeechSegment:
|
||||
words = list(getattr(segment, "words", None) or [])
|
||||
return SpeechSegment(
|
||||
text=segment.text.strip(),
|
||||
start=_seconds(segment.start),
|
||||
end=_seconds(segment.end),
|
||||
words=tuple(
|
||||
SpokenWord(
|
||||
word=spoken.word.strip(),
|
||||
start=_seconds(spoken.start),
|
||||
end=_seconds(spoken.end),
|
||||
confidence=float(getattr(spoken, "probability", 0.0) or 0.0),
|
||||
)
|
||||
for spoken in words
|
||||
),
|
||||
confidence=_confidence(words),
|
||||
)
|
||||
|
||||
|
||||
class SpeechTranscriber:
|
||||
"""Whisper on CTranslate2: 89 seconds of speech in 1.5 seconds of CPU, int8."""
|
||||
|
||||
def __init__(self, settings: Settings | None = None) -> None:
|
||||
self.settings = settings or get_settings()
|
||||
self._model: Any = None
|
||||
|
||||
def model_dir(self) -> Path | None:
|
||||
return self.settings.asr_model_dir
|
||||
|
||||
def available(self) -> bool:
|
||||
directory = self.model_dir()
|
||||
return directory is not None and (directory / MODEL_GRAPH).exists()
|
||||
|
||||
def transcribe(self, path: Path) -> list[SpeechSegment]:
|
||||
"""Words with the instant each was spoken; the timings are the point of the feature."""
|
||||
directory = self.model_dir()
|
||||
if directory is None:
|
||||
raise SystemExit(f"no speech model snapshot is configured; {DOWNLOAD_HINT}")
|
||||
if not (directory / MODEL_GRAPH).exists():
|
||||
raise SystemExit(f"{directory} holds no {MODEL_GRAPH}; {DOWNLOAD_HINT}")
|
||||
model = self._load(directory)
|
||||
segments, _info = model.transcribe(decode_audio(path), word_timestamps=True)
|
||||
return [_segment(segment) for segment in segments]
|
||||
|
||||
def _load(self, directory: Path) -> Any:
|
||||
if self._model is None:
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
self._model = WhisperModel(str(directory), device="cpu", compute_type="int8")
|
||||
return self._model
|
||||
Reference in new issue
Block a user