ZacharyZhang-NY/ely-eye
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make the speech numbers reproducible instead of merely printed

Browse Source 
A review of 5289cb4 found the ASR report was not honest arithmetic:

- "audio s" was the last segment's end, not the media length. The real
  container durations are 88.93 / 109.18 / 60.07 s, so the report understated
  the audio by 48 s and every speed cell was wrong.
- Decoding was unpinned, so a rerun was a new draw: WER moved 0.242 to 0.262
  and one video moved 0.278 to 0.583. Greedy single-temperature decoding with
  no carry-over now answers the same way twice, and the harness runs the pass
  twice to show it.
- The report claimed its WER was an upper bound. It is not a bound in either
  direction: a caption that drops words also hides words the model dropped.
  The denominators (201 / 296 / 36 reference words) are printed instead.
- "66 speech atoms" was bench-side arithmetic. The store now says 80 audio
  atoms carrying 563 word timestamps, read back after a real ingest.

Same silent-skip bug in a second place: a text file over max_text_file_bytes
returned an empty batch with no reason. It raises FileTooLarge with its size,
and a walk records that refusal like any other.

Also fixed from the same review: the Go search tool never published the audio
modality, cli_senses carried a second json_dump, the no-audio-track branch had
a test that fed it markdown, _seconds was an empty wrapper, and the docstring
quoted a speed no report measured.

Gate root cause: bench/tests imports the bench package by name with no root
conftest, so bare pytest could not collect it at all. pythonpath in pyproject
fixes both launch styles; 212 tests pass either way.

Verification: 212 passed (bare pytest and python -m pytest), ruff clean,
mypy clean on the six touched files with a cold cache, repo-wide mypy 24/6
unchanged from HEAD, pyright 0 errors on the touched files, go build/vet/test
and gofmt clean.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 22:23:33 -04:00
1 parent 5289cb4b01
commit 709104e809
9 files changed
+306 -92
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bench/asr_check.py
+138 -58
View File
@@ -1,8 +1,9 @@
"""Measure what the speech reader hears, against subtitles nobody in this repo wrote.
Video-MME ships community subtitles for its videos. They are third-party text, so they are a
fair reference for word error rate, with one caveat the report states: a caption track is not
a verbatim transcript, so the number is an upper bound on the model's real error rate.
Video-MME ships community subtitles. They are third-party text, so they are a usable reference
for word error rate. They are not a verbatim transcript either, and the gap cuts both ways: a
caption that drops words also hides words the model dropped. The report states the denominator
instead of claiming a bound.
"""
from __future__ import annotations
@@ -11,13 +12,23 @@ import hashlib
import json
import re
import subprocess
import tempfile
import time
from itertools import pairwise
from pathlib import Path
from typing import Any
from ely_eye.asr import SpeechSegment, SpeechTranscriber, SpokenWord
from ely_eye.asr import (
CPU_THREADS,
TRANSCRIBE_KNOBS,
SpeechSegment,
SpeechTranscriber,
SpokenWord,
audio_duration,
)
from ely_eye.config import Settings
from ely_eye.db import Database
from ely_eye.ingestion import IngestionService
from ely_eye.schemas import Modality
from bench.systems import machine_facts
@@ -27,6 +38,7 @@ ROOT = Path(__file__).resolve().parent
CORPUS = ROOT / "corpus" / "lmms-eval" / "Video-MME"
MODEL = ROOT / "models" / "faster-whisper-tiny"
REPORTS = ROOT / "reports"
PASSES = 2
PUNCTUATION = re.compile(r"[^\w' ]+")
TAGS = re.compile(r"<[^>]+>")
INDEX_LINE = re.compile(r"^\d+$")
@@ -92,70 +104,120 @@ def check_timings(segments: list[SpeechSegment]) -> dict[str, Any]:
return {"words": len(words), "out_of_order": out_of_order, "before_their_segment": outside}
def measure_once(transcriber: SpeechTranscriber, pairs: list[tuple[Path, Path]]) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for video, subtitle in pairs:
seconds = audio_duration(video)
if not seconds:
raise SystemExit(f"{video}: the container reports no duration, so the speed claim cannot be measured")
started = time.time()
segments = transcriber.transcribe(video)
took = time.time() - started
hyp_words = normalise(" ".join(segment.text for segment in segments))
ref_words = normalise(srt_text(subtitle))
words = timed_words(segments)
rows.append(
{
"video": video.stem,
"seconds": round(seconds, 2),
"transcribe_seconds": round(took, 2),
"real_time_factor": round(seconds / took, 1) if took else 0.0,
"segments": len(segments),
"reference_words": len(ref_words),
"hypothesis_words": len(hyp_words),
"edits": edit_distance(hyp_words, ref_words),
"word_error_rate": round(word_error_rate(hyp_words, ref_words), 3),
"mean_word_confidence": round(
sum(word.confidence for word in words) / max(len(words), 1), 3
),
"transcript": " ".join(segment.text for segment in segments),
**check_timings(segments),
}
)
return rows
def speech_atoms_in_the_store(videos: list[Path]) -> dict[str, Any]:
"""Ingest for real, then read the store back: the claim is about the product, not the model."""
with tempfile.TemporaryDirectory() as scratch:
settings = Settings(home=Path(scratch) / "home", asr_model_dir=MODEL)
settings.ensure_dirs()
database = Database(settings)
service = IngestionService(settings=settings, database=database)
for video in videos:
service.ingest_path(video)
atoms = database.list_atoms()
audio = [atom for atom in atoms if atom.modality == Modality.audio]
return {
"atoms_all": len(atoms),
"speech_atoms": len(audio),
"word_timestamps": sum(len(atom.metadata.get("words", [])) for atom in audio),
"sources": len({atom.source_id for atom in audio}),
}
def micro_wer(rows: list[dict[str, Any]]) -> float:
reference = sum(row["reference_words"] for row in rows)
if not reference:
raise SystemExit("the subtitle files produced no reference words; the parser or the files are wrong")
return round(sum(row["edits"] for row in rows) / reference, 3)
def main() -> None:
if not (MODEL / "model.bin").exists():
raise SystemExit(f"{MODEL / 'model.bin'} is missing; fetch Systran/faster-whisper-tiny into {MODEL}")
videos = sorted((CORPUS / "videos").glob("*.mp4"))
if not videos:
raise SystemExit(f"no videos under {CORPUS / 'videos'}; unpack them from videos_chunked_01.zip")
pairs = [(video, CORPUS / "subtitles" / f"{video.stem}.srt") for video in videos]
pairs = [(video, srt) for video, srt in pairs if srt.exists()]
if not pairs:
raise SystemExit(f"none of {len(videos)} videos has a subtitle in {CORPUS / 'subtitles'}")
transcriber = SpeechTranscriber(Settings(asr_model_dir=MODEL))
rows: list[dict[str, Any]] = []
for video in videos:
subtitle = CORPUS / "subtitles" / f"{video.stem}.srt"
if not subtitle.exists():
continue
started = time.time()
segments = transcriber.transcribe(video)
took = time.time() - started
hypothesis = " ".join(segment.text for segment in segments)
reference = srt_text(subtitle)
hyp_words, ref_words = normalise(hypothesis), normalise(reference)
audio_seconds = max((segment.end for segment in segments), default=0.0)
rows.append(
{
"video": video.stem,
"seconds": round(audio_seconds, 1),
"transcribe_seconds": round(took, 2),
"real_time_factor": round(audio_seconds / took, 1) if took else 0.0,
"segments": len(segments),
"speech_atoms": sum(1 for segment in segments if segment.text),
"modality": Modality.audio.value,
"hypothesis_words": len(hyp_words),
"reference_words": len(ref_words),
"word_error_rate": round(word_error_rate(hyp_words, ref_words), 3),
"mean_word_confidence": round(
sum(word.confidence for word in timed_words(segments)) / max(len(timed_words(segments)), 1), 3
),
**check_timings(segments),
}
)
if not rows:
raise SystemExit(f"none of {len(videos)} videos has a subtitle in {CORPUS / 'subtitles'}")
reference_words = sum(row["reference_words"] for row in rows)
if not reference_words:
raise SystemExit("the subtitle files produced no reference words; the parser or the files are wrong")
passes = [measure_once(transcriber, pairs) for _ in range(PASSES)]
rows, repeat = passes[0], passes[-1]
store = speech_atoms_in_the_store(videos)
if not store["speech_atoms"]:
raise SystemExit("ingestion produced no audio atoms; the product path is broken even if the model works")
totals = {
"videos": len(rows),
"audio_seconds": round(sum(row["seconds"] for row in rows), 1),
"word_error_rate": round(
sum(row["word_error_rate"] * row["reference_words"] for row in rows) / reference_words,
3,
),
"speech_atoms": sum(row["speech_atoms"] for row in rows),
"audio_seconds": round(sum(row["seconds"] for row in rows), 2),
"transcribe_seconds": round(sum(row["transcribe_seconds"] for row in rows), 2),
"word_error_rate": micro_wer(rows),
"reference_words": sum(row["reference_words"] for row in rows),
"hypothesis_words": sum(row["hypothesis_words"] for row in rows),
"words": sum(row["words"] for row in rows),
"out_of_order": sum(row["out_of_order"] for row in rows),
"before_their_segment": sum(row["before_their_segment"] for row in rows),
**store,
}
report = {
"generated_utc": time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime()),
"commit": git_sha(),
"model": {"id": "Systran/faster-whisper-tiny", "sha256_model_bin": model_sha256(), "compute": "cpu int8"},
"reference": "Video-MME community subtitles (third-party captions, not verbatim transcripts)",
"model": {
"id": "Systran/faster-whisper-tiny",
"sha256_model_bin": model_sha256(),
"compute": "cpu int8",
"cpu_threads": CPU_THREADS,
"decode_knobs": {key: str(value) for key, value in TRANSCRIBE_KNOBS.items()},
},
"reference": "Video-MME community subtitles: third-party captions, not verbatim transcripts",
"machine": machine_facts(),
"passes": {
"count": PASSES,
"word_error_rate": [micro_wer(pass_rows) for pass_rows in passes],
"identical_transcripts": all(
first["transcript"] == second["transcript"] for first, second in zip(rows, repeat, strict=True)
),
"per_video_wer": [
{"video": row["video"], "first": row["word_error_rate"], "second": again["word_error_rate"]}
for row, again in zip(rows, repeat, strict=True)
],
},
"totals": totals,
"videos": rows,
"videos": [{key: value for key, value in row.items() if key != "transcript"} for row in rows],
}
REPORTS.mkdir(exist_ok=True)
(REPORTS / "asr.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
@@ -164,36 +226,54 @@ def main() -> None:
def write_markdown(report: dict[str, Any]) -> None:
totals, model = report["totals"], report["model"]
totals, model, passes = report["totals"], report["model"], report["passes"]
lines = [
f"# ASR: what the speech reader hears (commit {report['commit'][:9]})",
"",
f"- generated {report['generated_utc']}",
f"- model: {model['id']} ({model['compute']}), model.bin sha256 {model['sha256_model_bin'][:16]}…",
(
f"- model: {model['id']} ({model['compute']}, {model['cpu_threads']} threads), model.bin sha256 "
f"{model['sha256_model_bin'][:16]}…"
),
f"- decode: {', '.join(f'{key}={value}' for key, value in model['decode_knobs'].items())}",
f"- reference: {report['reference']}",
f"- machine: {report['machine']}",
"",
"| video | audio s | transcribe s | ×realtime | speech atoms | words | WER | mean word confidence |",
"| video | audio s | transcribe s | ×realtime | reference words | words | WER | mean word confidence |",
"| --- | --- | --- | --- | --- | --- | --- | --- |",
]
for row in report["videos"]:
lines.append(
f"| {row['video']} | {row['seconds']} | {row['transcribe_seconds']} | {row['real_time_factor']} "
f"| {row['speech_atoms']} | {row['words']} | {row['word_error_rate']} | {row['mean_word_confidence']} |"
f"| {row['reference_words']} | {row['words']} | {row['word_error_rate']} "
f"| {row['mean_word_confidence']} |"
)
lines += [
"",
(
f"Over {totals['videos']} videos and {totals['audio_seconds']} seconds of audio: word error rate "
f"**{totals['word_error_rate']}** against captions nobody here wrote, {totals['speech_atoms']} speech "
f"atoms carrying {totals['words']} word-level timestamps, {totals['out_of_order']} words out of order "
f"and {totals['before_their_segment']} starting before their own segment."
f"**{totals['word_error_rate']}** over {totals['reference_words']} reference words "
f"({totals['hypothesis_words']} words heard), transcribed in {totals['transcribe_seconds']} seconds."
),
"",
(
"A caption track drops words a transcript keeps, so this WER is an upper bound on the model's real "
"error. The point of the run is not the number: before this change an audio file produced no memory "
"at all, and every word here now has an instant attached to it."
f"Repeating the pass on the same machine gave WER {' then '.join(str(value) for value in passes['word_error_rate'])}"
f"; the transcripts were {'' if passes['identical_transcripts'] else 'not '}byte-identical. "
"Decoding is greedy with no retry ladder, so a rerun is the same answer, not a new draw."
),
"",
(
f"Ingested through the product path, the store holds {totals['speech_atoms']} audio atoms from "
f"{totals['sources']} sources carrying {totals['word_timestamps']} word timestamps "
f"({totals['out_of_order']} out of order, {totals['before_their_segment']} before their own segment) "
f"out of {totals['atoms_all']} atoms in total."
),
"",
(
"The captions are not a transcript, so this WER is not a bound in either direction: a caption that "
"drops words also hides words the model dropped. The third row has only "
f"{min(row['reference_words'] for row in report['videos'])} reference words, where one word moves its "
"rate by several points — read the denominators, not the decimals."
),
]
(REPORTS / "ASR.md").write_text("\n".join(lines) + "\n", encoding="utf-8")