diff --git a/CHECKLIST.md b/CHECKLIST.md index dbd126e..772affd 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -156,11 +156,19 @@ - [x] **p1-2** 音频摄入 - [x] p1-2.1 `Modality` 加 audio；未知扩展名不再静默跳过 - [x] p1-2.2 ASR 接入，转写 atom 带词级时间戳 - - 验收：`pytest backend/tests/test_asr.py` 绿（8 个测试）；ASR **实跑**，不是 DEFERRED - - 实测证据（`bench/reports/ASR.md`，commit 03ca357）：3 段 Video-MME 真视频 210.5 秒音频， - 对着第三方字幕 WER **0.242**，CPU int8 **51× 实时**，产出 66 个 speech atom、550 个词级时间戳， - 0 个词时序倒错、0 个词早于自己所属片段。模型 `Systran/faster-whisper-tiny`（78.2MB， - model.bin sha256 dcb76c6586fc06cb…），远低于 500MB 上限。字幕不是逐字稿，所以 WER 是上界。 + - 验收：`pytest backend/tests/test_asr.py` 绿（13 个测试）；ASR **实跑**，不是 DEFERRED + - 实测证据（`bench/reports/ASR.md`，`.venv/bin/python -m bench.asr_check` 生成）：3 段 Video-MME 真视频 + **258.18 秒**音频（容器时长 88.93 / 109.18 / 60.07），对着第三方字幕 WER **0.242**——分母是 + **533 个参考词**（听到 563 个词），三行各自的参考词数是 201 / 296 / 36。CPU int8 4 线程 + 3.55 秒转完，逐行 61.6× / 73.0× / 97.7× 实时。走产品路径摄入后从库里读回：**80 个 audio atom**、 + **563 个词级时间戳**、0 个时序倒错、0 个早于自己所属片段（总共 133 个 atom）。模型 + `Systran/faster-whisper-tiny`（78.2MB，model.bin sha256 dcb76c6586fc06cb…），远低于 500MB 上限。 + - 可复现性是量出来的：解码参数钉死 `beam_size=1, temperature=(0.0,), condition_on_previous_text=False, + vad_filter=False, cpu_threads=4`（`temperature` 给单元素 tuple 才是真的没有回退阶梯—— + faster-whisper 1.2.1 **没有** `temperature_increment_on_fallback` 这个参数，第一版我传了它， + 直接 `TypeError`）。同一台机器连跑两遍：WER 0.242 → 0.242，转写文本逐字节相同。 + - 字幕不是逐字稿，所以这个 WER **不是任何方向的界**：字幕漏词也会掩盖模型漏词。 + 报告把分母写在表里，不写「上界」这种断言。 - 偏差与根因： - 验收路径 `tests/test_audio_ingest.py` 不存在，实际是 `backend/tests/test_asr.py`。 - `Modality` 只加 `audio`，没加 `transcript`：一份转写就是带时间戳的 audio atom， @@ -172,16 +180,33 @@ 真实基类是 `av.error.FFmpegError`；只 catch 它，我自己的 `ValueError` 消息才不会被吞。 - 未知扩展名：显式给路径 → 抛 `UnsupportedFile`（“no reader for .xyz”）；目录扫描 → 记进 `IngestResponse.skipped` 带原因，继续摄入能读的。以前是返回空 batch，无声无息。 + - 同一个静默毛病还有一处被 review 抓出来：`_ingest_text` 对超过 `max_text_file_bytes`（10 MiB） + 的文件也是直接返回空 batch。本仓库自己的语料里就有 277MB 的 `longmemeval_s_cleaned.json`。 + 现在抛 `FileTooLarge`（带字节数和上限），扫描照样记进 `skipped`。 - 视频现在除了帧 atom 还会产出 speech atom（同一个 mp4：18 帧 + 31 段语音）。 - 顺手把 p2-4 提前做了：`ingestion.py` 748 行超了 500 规则，拆成 - `ingestion_common.py`(99) / `ingestion_media.py`(ABC mixin, 285) / `ingestion_repository.py`(346) / - `ingestion.py`(193)；`cli.py` 544 行也超了，按已有的 `cli_transactive.register` 模式拆出 - `cli_senses.py`(84) 和 `cli_common.py`(json_dump)，`cli.py` 现在 479 行。 + `ingestion_common.py`(102) / `ingestion_media.py`(ABC mixin, 278) / `ingestion_repository.py`(321) / + `ingestion.py`(177)；`cli.py` 544 行也超了，按已有的 `cli_transactive.register` 模式拆出 + `cli_senses.py`(79) 和 `cli_common.py`(json_dump)，`cli.py` 现在 479 行。 + - review 第一轮 FAIL 的 11 条全部落地：钉死解码参数并跑两遍证明确定性；用容器真实时长 + 而不是「最后一段的结束时间」；表里露出参考词分母、删掉「上界」断言；speech atom 数改成 + 从库里读回（80，不是 bench 自己数的 66）；报告在代码提交之后重新生成，stamp 指向含代码的 + commit；`cli_senses` 里重复的 `json_dump` 删掉改用 `cli_common`；Go 的 `search_atoms` 描述补上 + `audio`（MCP 调用方原本发现不了这个过滤值）；`test_a_file_with_no_audio_track_says_so` 原本 + 测的是 markdown，现在用 PyAV 写真·无音轨 mp4 命中 `asr.py` 那条分支，另加空 wav 命中 + 「音轨是空的」；删掉 `_seconds` 这个空壳 wrapper。 + - 顺手改了个默认值（review 也点了）：`config.py` 的 `embedding_model_id` 从 + `Qwen/Qwen3-VL-Embedding-8B`（16GB，违反本轮 ≤500MB 的界）换成 + `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2`——冻结的 bench 数字用的就是它。 + - 门禁里挖出的真根因：`bench/tests/*` 用 `from bench.x import` 而仓库根没有 conftest， + 所以裸 `pytest` 会 `ModuleNotFoundError: No module named 'bench'`（只有 `python -m pytest` 能跑）。 + `pyproject.toml` 加 `[tool.pytest.ini_options] pythonpath=["."] testpaths=[…]`，两种起法都绿。 - 拆完顺手修掉搬过来的旧类型债：`MediaIngest._source_record` 改成 `@abstractmethod` （半接好的 host 现在在构造时就失败，不会静默返回 None）、PDF 页循环与 bbox 减法、 `page.get_text` 的 union 收窄；`pyproject.toml` 里失效的 `fitz.*` mypy override 换成 `faster_whisper.*`。 - - 全仓 mypy 从 29 个错降到 **24**（本任务净修 5 个，没新增）；改动文件 pyright **0 错**、 - ruff 干净、207 个测试绿。 + - 全仓 mypy：干净树上冷缓存实测 parent `03ca357` **26** 个错 → 本任务 **24**（净修 2 个， + 没新增；之前写的 29 是在半成品工作树上量的，不算）；改动文件 pyright **0 错**、 + ruff 干净、**212** 个测试绿（裸 `pytest` 与 `python -m pytest` 都是 212）。 - [ ] **p1-3** adapter 加载 - [ ] p1-3.1 按卡带/场景解析 6 个 adapter，不再只有 `runtime.py:317` - [ ] p1-3.2 加载结果可观测：日志 + `GET /api/runtime/adapters` diff --git a/backend/src/ely_eye/asr.py b/backend/src/ely_eye/asr.py index 755c7ef..161410e 100644 --- a/backend/src/ely_eye/asr.py +++ b/backend/src/ely_eye/asr.py @@ -17,6 +17,14 @@ DOWNLOAD_HINT = ( DEFERRED = "deferred: no speech model snapshot" MODEL_GRAPH = "model.bin" SAMPLE_RATE = 16_000 +CPU_THREADS = 4 +TRANSCRIBE_KNOBS: dict[str, Any] = { + "word_timestamps": True, + "beam_size": 1, + "temperature": (0.0,), + "condition_on_previous_text": False, + "vad_filter": False, +} @dataclass(frozen=True) @@ -66,8 +74,18 @@ def decode_audio(path: Path) -> np.ndarray: return np.concatenate(frames).astype(np.float32) -def _seconds(value: Any) -> float: - return float(value) +def audio_duration(path: Path) -> float: + """Seconds of media the container claims; 0.0 when it does not say.""" + import av + from av.error import FFmpegError + + try: + with av.open(str(path)) as container: + if not container.duration: + return 0.0 + return container.duration / av.time_base + except FFmpegError as error: + raise ValueError(f"{path}: the container cannot be read ({error})") from error def _confidence(words: list[Any]) -> float: @@ -81,13 +99,13 @@ def _segment(segment: Any) -> SpeechSegment: words = list(getattr(segment, "words", None) or []) return SpeechSegment( text=segment.text.strip(), - start=_seconds(segment.start), - end=_seconds(segment.end), + start=float(segment.start), + end=float(segment.end), words=tuple( SpokenWord( word=spoken.word.strip(), - start=_seconds(spoken.start), - end=_seconds(spoken.end), + start=float(spoken.start), + end=float(spoken.end), confidence=float(getattr(spoken, "probability", 0.0) or 0.0), ) for spoken in words @@ -97,7 +115,7 @@ def _segment(segment: Any) -> SpeechSegment: class SpeechTranscriber: - """Whisper on CTranslate2: 89 seconds of speech in 1.5 seconds of CPU, int8.""" + """Whisper on CTranslate2: 88.9 seconds of speech in 1.7 seconds of CPU int8 (bench/reports/ASR.md).""" def __init__(self, settings: Settings | None = None) -> None: self.settings = settings or get_settings() @@ -118,12 +136,15 @@ class SpeechTranscriber: if not (directory / MODEL_GRAPH).exists(): raise SystemExit(f"{directory} holds no {MODEL_GRAPH}; {DOWNLOAD_HINT}") model = self._load(directory) - segments, _info = model.transcribe(decode_audio(path), word_timestamps=True) + # Greedy, no retry ladder, no carry-over: the same file must answer the same way twice. + segments, _info = model.transcribe(decode_audio(path), **TRANSCRIBE_KNOBS) return [_segment(segment) for segment in segments] def _load(self, directory: Path) -> Any: if self._model is None: from faster_whisper import WhisperModel - self._model = WhisperModel(str(directory), device="cpu", compute_type="int8") + self._model = WhisperModel( + str(directory), device="cpu", compute_type="int8", cpu_threads=CPU_THREADS + ) return self._model diff --git a/backend/src/ely_eye/cli_senses.py b/backend/src/ely_eye/cli_senses.py index 616209e..44ad1d5 100644 --- a/backend/src/ely_eye/cli_senses.py +++ b/backend/src/ely_eye/cli_senses.py @@ -9,12 +9,12 @@ from __future__ import annotations from collections.abc import Callable from dataclasses import asdict from pathlib import Path -from typing import Any import typer from .asr import DEFERRED as SPEECH_DEFERRED from .asr import SpeechTranscriber +from .cli_common import json_dump from .config import Settings from .db import Database from .describe import DEFERRED, VisionDescriber @@ -54,11 +54,6 @@ def senses_report(settings: Settings, db: Database) -> dict[str, dict[str, objec def register(app: typer.Typer, settings: Callable[[], Settings]) -> None: """Add the media commands. The settings thunk lets one patch in a test redirect every command.""" - def json_dump(value: Any) -> str: - import json - - return json.dumps(value, ensure_ascii=False, indent=2) - @app.command() def transcribe(path: Path) -> None: """Words with the instant each was spoken; the snapshot stays the operator's to supply.""" diff --git a/backend/src/ely_eye/ingestion.py b/backend/src/ely_eye/ingestion.py index de450a9..05e42a3 100644 --- a/backend/src/ely_eye/ingestion.py +++ b/backend/src/ely_eye/ingestion.py @@ -16,6 +16,7 @@ from .ingestion_common import ( SUPPORTED_IMAGE_EXTENSIONS, SUPPORTED_TEXT_EXTENSIONS, SUPPORTED_VIDEO_EXTENSIONS, + FileTooLarge, IngestBatch, UnsupportedFile, ) @@ -62,7 +63,7 @@ class IngestionService(MediaIngest, RepositoryIngest): for file_path in files: try: file_batch = self._ingest_file(file_path) - except UnsupportedFile as error: + except (UnsupportedFile, FileTooLarge) as error: if explicit: raise skipped.append(SkippedFile(path=str(file_path), reason=str(error))) @@ -147,8 +148,10 @@ class IngestionService(MediaIngest, RepositoryIngest): ) def _ingest_text(self, path: Path, modality: Modality) -> IngestBatch: - if path.stat().st_size > self.settings.max_text_file_bytes: - return IngestBatch(sources=[], atoms=[]) + limit = self.settings.max_text_file_bytes + size = path.stat().st_size + if size > limit: + raise FileTooLarge(f"{path} is {size} bytes, over max_text_file_bytes={limit}") source = self._source_record(path, modality.value) text = path.read_text(encoding="utf-8", errors="replace") atom = EvidenceAtom( diff --git a/backend/src/ely_eye/ingestion_common.py b/backend/src/ely_eye/ingestion_common.py index 8094034..3bad277 100644 --- a/backend/src/ely_eye/ingestion_common.py +++ b/backend/src/ely_eye/ingestion_common.py @@ -54,6 +54,10 @@ class UnsupportedFile(ValueError): """No reader for this file: a walk records the reason, an explicit path fails.""" +class FileTooLarge(ValueError): + """A reader exists but the file is over the configured cap: refused with its size stated.""" + + @dataclass class IngestBatch: sources: list[SourceRecord] diff --git a/backend/tests/test_asr.py b/backend/tests/test_asr.py index 842fc8c..2bf1208 100644 --- a/backend/tests/test_asr.py +++ b/backend/tests/test_asr.py @@ -2,6 +2,7 @@ from __future__ import annotations +import wave from pathlib import Path from typing import Any, cast @@ -9,10 +10,19 @@ import numpy as np import pytest from ely_eye import ingestion_media as media_module -from ely_eye.asr import DEFERRED, SpeechSegment, SpeechTranscriber, SpokenWord, decode_audio +from ely_eye.asr import ( + DEFERRED, + SAMPLE_RATE, + SpeechSegment, + SpeechTranscriber, + SpokenWord, + audio_duration, + decode_audio, +) from ely_eye.config import Settings from ely_eye.db import Database from ely_eye.ingestion import IngestionService, UnsupportedFile +from ely_eye.ingestion_common import FileTooLarge from ely_eye.ocr import OcrResult from ely_eye.schemas import Modality @@ -89,12 +99,18 @@ class FakeCv2: return True, FakeEncoded() -def _settings(tmp_path: Path, asr_model_dir: Path | None = None) -> Settings: - return Settings(home=tmp_path / "home", asr_model_dir=asr_model_dir) +def _settings(tmp_path: Path, asr_model_dir: Path | None = None, max_text_file_bytes: int = 10_000_000) -> Settings: + return Settings(home=tmp_path / "home", asr_model_dir=asr_model_dir, max_text_file_bytes=max_text_file_bytes) -def _service(tmp_path: Path, transcriber: Any) -> tuple[IngestionService, Database]: - settings = _settings(tmp_path) +def _service( + tmp_path: Path, + transcriber: Any, + *, + asr_model_dir: Path | None = None, + max_text_file_bytes: int = 10_000_000, +) -> tuple[IngestionService, Database]: + settings = _settings(tmp_path, asr_model_dir=asr_model_dir, max_text_file_bytes=max_text_file_bytes) settings.ensure_dirs() database = Database(settings) service = IngestionService(settings=settings, database=database) @@ -202,7 +218,71 @@ def test_the_transcriber_names_the_snapshot_it_wants(tmp_path: Path) -> None: half.transcribe(_audio(tmp_path)) +def _video_only(path: Path) -> Path: + """A real mp4 with a video stream and no audio track, written by the decoder we already ship.""" + import av + + with av.open(str(path), "w") as container: + stream = container.add_stream("mpeg4", rate=10) + stream.width, stream.height, stream.pix_fmt = 64, 64, "yuv420p" + for _ in range(5): + frame = av.VideoFrame.from_ndarray(np.zeros((64, 64, 3), dtype=np.uint8), format="rgb24") + for packet in stream.encode(frame): + container.mux(packet) + for packet in stream.encode(): + container.mux(packet) + return path + + def test_a_file_with_no_audio_track_says_so(tmp_path: Path) -> None: + video = _video_only(tmp_path / "silent.mp4") + + with pytest.raises(ValueError, match="the file holds no audio track"): + decode_audio(video) + + +def test_an_audio_track_with_no_frames_says_so(tmp_path: Path) -> None: + silent = tmp_path / "silent.wav" + with wave.open(str(silent), "wb") as writer: + writer.setnchannels(1) + writer.setsampwidth(2) + writer.setframerate(SAMPLE_RATE) + + with pytest.raises(ValueError, match="the audio track is empty"): + decode_audio(silent) + + +def test_a_walk_records_a_file_over_the_size_cap(tmp_path: Path) -> None: + service, database = _service(tmp_path, FakeTranscriber(available=False), max_text_file_bytes=32) + folder = tmp_path / "shelf" + folder.mkdir() + (folder / "note.md").write_text("# 钥匙放在抽屉里\n", encoding="utf-8") + (folder / "huge.json").write_text('{"钥匙": "抽屉"}' * 4, encoding="utf-8") + + response = service.ingest_path(folder) + + assert response.atom_count > 0, "the small file is still ingested" + assert [item.path for item in response.skipped] == [str(folder / "huge.json")] + assert "over max_text_file_bytes=32" in response.skipped[0].reason + assert database.atom_count() == response.atom_count + + +def test_an_explicit_file_over_the_size_cap_fails_loudly(tmp_path: Path) -> None: + service, _database = _service(tmp_path, FakeTranscriber(available=False), max_text_file_bytes=16) + huge = tmp_path / "huge.json" + huge.write_text('{"钥匙": "抽屉"}' * 4, encoding="utf-8") + + with pytest.raises(FileTooLarge, match="over max_text_file_bytes=16"): + service.ingest_path(huge) + + +def test_the_container_duration_is_the_media_length(tmp_path: Path) -> None: + video = _video_only(tmp_path / "silent.mp4") + + assert audio_duration(video) == pytest.approx(0.5, abs=0.05) + + +def test_a_text_file_cannot_be_decoded(tmp_path: Path) -> None: markdown = tmp_path / "notes.md" markdown.write_text("# 钥匙\n", encoding="utf-8") diff --git a/bench/asr_check.py b/bench/asr_check.py index c5a8c58..741435f 100644 --- a/bench/asr_check.py +++ b/bench/asr_check.py @@ -1,8 +1,9 @@ """Measure what the speech reader hears, against subtitles nobody in this repo wrote. -Video-MME ships community subtitles for its videos. They are third-party text, so they are a -fair reference for word error rate, with one caveat the report states: a caption track is not -a verbatim transcript, so the number is an upper bound on the model's real error rate. +Video-MME ships community subtitles. They are third-party text, so they are a usable reference +for word error rate. They are not a verbatim transcript either, and the gap cuts both ways: a +caption that drops words also hides words the model dropped. The report states the denominator +instead of claiming a bound. """ from __future__ import annotations @@ -11,13 +12,23 @@ import hashlib import json import re import subprocess +import tempfile import time from itertools import pairwise from pathlib import Path from typing import Any -from ely_eye.asr import SpeechSegment, SpeechTranscriber, SpokenWord +from ely_eye.asr import ( + CPU_THREADS, + TRANSCRIBE_KNOBS, + SpeechSegment, + SpeechTranscriber, + SpokenWord, + audio_duration, +) from ely_eye.config import Settings +from ely_eye.db import Database +from ely_eye.ingestion import IngestionService from ely_eye.schemas import Modality from bench.systems import machine_facts @@ -27,6 +38,7 @@ ROOT = Path(__file__).resolve().parent CORPUS = ROOT / "corpus" / "lmms-eval" / "Video-MME" MODEL = ROOT / "models" / "faster-whisper-tiny" REPORTS = ROOT / "reports" +PASSES = 2 PUNCTUATION = re.compile(r"[^\w' ]+") TAGS = re.compile(r"<[^>]+>") INDEX_LINE = re.compile(r"^\d+$") @@ -92,70 +104,120 @@ def check_timings(segments: list[SpeechSegment]) -> dict[str, Any]: return {"words": len(words), "out_of_order": out_of_order, "before_their_segment": outside} +def measure_once(transcriber: SpeechTranscriber, pairs: list[tuple[Path, Path]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for video, subtitle in pairs: + seconds = audio_duration(video) + if not seconds: + raise SystemExit(f"{video}: the container reports no duration, so the speed claim cannot be measured") + started = time.time() + segments = transcriber.transcribe(video) + took = time.time() - started + hyp_words = normalise(" ".join(segment.text for segment in segments)) + ref_words = normalise(srt_text(subtitle)) + words = timed_words(segments) + rows.append( + { + "video": video.stem, + "seconds": round(seconds, 2), + "transcribe_seconds": round(took, 2), + "real_time_factor": round(seconds / took, 1) if took else 0.0, + "segments": len(segments), + "reference_words": len(ref_words), + "hypothesis_words": len(hyp_words), + "edits": edit_distance(hyp_words, ref_words), + "word_error_rate": round(word_error_rate(hyp_words, ref_words), 3), + "mean_word_confidence": round( + sum(word.confidence for word in words) / max(len(words), 1), 3 + ), + "transcript": " ".join(segment.text for segment in segments), + **check_timings(segments), + } + ) + return rows + + +def speech_atoms_in_the_store(videos: list[Path]) -> dict[str, Any]: + """Ingest for real, then read the store back: the claim is about the product, not the model.""" + with tempfile.TemporaryDirectory() as scratch: + settings = Settings(home=Path(scratch) / "home", asr_model_dir=MODEL) + settings.ensure_dirs() + database = Database(settings) + service = IngestionService(settings=settings, database=database) + for video in videos: + service.ingest_path(video) + atoms = database.list_atoms() + audio = [atom for atom in atoms if atom.modality == Modality.audio] + return { + "atoms_all": len(atoms), + "speech_atoms": len(audio), + "word_timestamps": sum(len(atom.metadata.get("words", [])) for atom in audio), + "sources": len({atom.source_id for atom in audio}), + } + + +def micro_wer(rows: list[dict[str, Any]]) -> float: + reference = sum(row["reference_words"] for row in rows) + if not reference: + raise SystemExit("the subtitle files produced no reference words; the parser or the files are wrong") + return round(sum(row["edits"] for row in rows) / reference, 3) + + def main() -> None: if not (MODEL / "model.bin").exists(): raise SystemExit(f"{MODEL / 'model.bin'} is missing; fetch Systran/faster-whisper-tiny into {MODEL}") videos = sorted((CORPUS / "videos").glob("*.mp4")) if not videos: raise SystemExit(f"no videos under {CORPUS / 'videos'}; unpack them from videos_chunked_01.zip") + pairs = [(video, CORPUS / "subtitles" / f"{video.stem}.srt") for video in videos] + pairs = [(video, srt) for video, srt in pairs if srt.exists()] + if not pairs: + raise SystemExit(f"none of {len(videos)} videos has a subtitle in {CORPUS / 'subtitles'}") transcriber = SpeechTranscriber(Settings(asr_model_dir=MODEL)) - rows: list[dict[str, Any]] = [] - for video in videos: - subtitle = CORPUS / "subtitles" / f"{video.stem}.srt" - if not subtitle.exists(): - continue - started = time.time() - segments = transcriber.transcribe(video) - took = time.time() - started - hypothesis = " ".join(segment.text for segment in segments) - reference = srt_text(subtitle) - hyp_words, ref_words = normalise(hypothesis), normalise(reference) - audio_seconds = max((segment.end for segment in segments), default=0.0) - rows.append( - { - "video": video.stem, - "seconds": round(audio_seconds, 1), - "transcribe_seconds": round(took, 2), - "real_time_factor": round(audio_seconds / took, 1) if took else 0.0, - "segments": len(segments), - "speech_atoms": sum(1 for segment in segments if segment.text), - "modality": Modality.audio.value, - "hypothesis_words": len(hyp_words), - "reference_words": len(ref_words), - "word_error_rate": round(word_error_rate(hyp_words, ref_words), 3), - "mean_word_confidence": round( - sum(word.confidence for word in timed_words(segments)) / max(len(timed_words(segments)), 1), 3 - ), - **check_timings(segments), - } - ) - if not rows: - raise SystemExit(f"none of {len(videos)} videos has a subtitle in {CORPUS / 'subtitles'}") - reference_words = sum(row["reference_words"] for row in rows) - if not reference_words: - raise SystemExit("the subtitle files produced no reference words; the parser or the files are wrong") + passes = [measure_once(transcriber, pairs) for _ in range(PASSES)] + rows, repeat = passes[0], passes[-1] + store = speech_atoms_in_the_store(videos) + if not store["speech_atoms"]: + raise SystemExit("ingestion produced no audio atoms; the product path is broken even if the model works") totals = { "videos": len(rows), - "audio_seconds": round(sum(row["seconds"] for row in rows), 1), - "word_error_rate": round( - sum(row["word_error_rate"] * row["reference_words"] for row in rows) / reference_words, - 3, - ), - "speech_atoms": sum(row["speech_atoms"] for row in rows), + "audio_seconds": round(sum(row["seconds"] for row in rows), 2), + "transcribe_seconds": round(sum(row["transcribe_seconds"] for row in rows), 2), + "word_error_rate": micro_wer(rows), + "reference_words": sum(row["reference_words"] for row in rows), + "hypothesis_words": sum(row["hypothesis_words"] for row in rows), "words": sum(row["words"] for row in rows), "out_of_order": sum(row["out_of_order"] for row in rows), "before_their_segment": sum(row["before_their_segment"] for row in rows), + **store, } report = { "generated_utc": time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime()), "commit": git_sha(), - "model": {"id": "Systran/faster-whisper-tiny", "sha256_model_bin": model_sha256(), "compute": "cpu int8"}, - "reference": "Video-MME community subtitles (third-party captions, not verbatim transcripts)", + "model": { + "id": "Systran/faster-whisper-tiny", + "sha256_model_bin": model_sha256(), + "compute": "cpu int8", + "cpu_threads": CPU_THREADS, + "decode_knobs": {key: str(value) for key, value in TRANSCRIBE_KNOBS.items()}, + }, + "reference": "Video-MME community subtitles: third-party captions, not verbatim transcripts", "machine": machine_facts(), + "passes": { + "count": PASSES, + "word_error_rate": [micro_wer(pass_rows) for pass_rows in passes], + "identical_transcripts": all( + first["transcript"] == second["transcript"] for first, second in zip(rows, repeat, strict=True) + ), + "per_video_wer": [ + {"video": row["video"], "first": row["word_error_rate"], "second": again["word_error_rate"]} + for row, again in zip(rows, repeat, strict=True) + ], + }, "totals": totals, - "videos": rows, + "videos": [{key: value for key, value in row.items() if key != "transcript"} for row in rows], } REPORTS.mkdir(exist_ok=True) (REPORTS / "asr.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") @@ -164,36 +226,54 @@ def main() -> None: def write_markdown(report: dict[str, Any]) -> None: - totals, model = report["totals"], report["model"] + totals, model, passes = report["totals"], report["model"], report["passes"] lines = [ f"# ASR: what the speech reader hears (commit {report['commit'][:9]})", "", f"- generated {report['generated_utc']}", - f"- model: {model['id']} ({model['compute']}), model.bin sha256 {model['sha256_model_bin'][:16]}…", + ( + f"- model: {model['id']} ({model['compute']}, {model['cpu_threads']} threads), model.bin sha256 " + f"{model['sha256_model_bin'][:16]}…" + ), + f"- decode: {', '.join(f'{key}={value}' for key, value in model['decode_knobs'].items())}", f"- reference: {report['reference']}", f"- machine: {report['machine']}", "", - "| video | audio s | transcribe s | ×realtime | speech atoms | words | WER | mean word confidence |", + "| video | audio s | transcribe s | ×realtime | reference words | words | WER | mean word confidence |", "| --- | --- | --- | --- | --- | --- | --- | --- |", ] for row in report["videos"]: lines.append( f"| {row['video']} | {row['seconds']} | {row['transcribe_seconds']} | {row['real_time_factor']} " - f"| {row['speech_atoms']} | {row['words']} | {row['word_error_rate']} | {row['mean_word_confidence']} |" + f"| {row['reference_words']} | {row['words']} | {row['word_error_rate']} " + f"| {row['mean_word_confidence']} |" ) lines += [ "", ( f"Over {totals['videos']} videos and {totals['audio_seconds']} seconds of audio: word error rate " - f"**{totals['word_error_rate']}** against captions nobody here wrote, {totals['speech_atoms']} speech " - f"atoms carrying {totals['words']} word-level timestamps, {totals['out_of_order']} words out of order " - f"and {totals['before_their_segment']} starting before their own segment." + f"**{totals['word_error_rate']}** over {totals['reference_words']} reference words " + f"({totals['hypothesis_words']} words heard), transcribed in {totals['transcribe_seconds']} seconds." ), "", ( - "A caption track drops words a transcript keeps, so this WER is an upper bound on the model's real " - "error. The point of the run is not the number: before this change an audio file produced no memory " - "at all, and every word here now has an instant attached to it." + f"Repeating the pass on the same machine gave WER {' then '.join(str(value) for value in passes['word_error_rate'])}" + f"; the transcripts were {'' if passes['identical_transcripts'] else 'not '}byte-identical. " + "Decoding is greedy with no retry ladder, so a rerun is the same answer, not a new draw." + ), + "", + ( + f"Ingested through the product path, the store holds {totals['speech_atoms']} audio atoms from " + f"{totals['sources']} sources carrying {totals['word_timestamps']} word timestamps " + f"({totals['out_of_order']} out of order, {totals['before_their_segment']} before their own segment) " + f"out of {totals['atoms_all']} atoms in total." + ), + "", + ( + "The captions are not a transcript, so this WER is not a bound in either direction: a caption that " + "drops words also hides words the model dropped. The third row has only " + f"{min(row['reference_words'] for row in report['videos'])} reference words, where one word moves its " + "rate by several points — read the denominators, not the decimals." ), ] (REPORTS / "ASR.md").write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/ely-eye-mcp/internal/mcpserver/tools.go b/ely-eye-mcp/internal/mcpserver/tools.go index 9fe0fdb..0404f5f 100644 --- a/ely-eye-mcp/internal/mcpserver/tools.go +++ b/ely-eye-mcp/internal/mcpserver/tools.go @@ -23,7 +23,7 @@ type GetCartridgeInput struct { type SearchAtomsInput struct { Query string `json:"query" jsonschema:"search query, matched by the server's own ranking"` - Modality string `json:"modality,omitempty" jsonschema:"optional exact modality filter such as text, code, image, pdf_page, pdf_region, video_frame, ui_screenshot"` + Modality string `json:"modality,omitempty" jsonschema:"optional exact modality filter such as text, code, image, audio, pdf_page, pdf_region, video_frame, ui_screenshot"` Holder string `json:"holder,omitempty" jsonschema:"optional: only this holder's memories may answer"` Scenario string `json:"scenario,omitempty" jsonschema:"optional: only this scenario, e.g. home or work"` Limit int `json:"limit,omitempty" jsonschema:"maximum number of atoms to return, capped at 100"` diff --git a/pyproject.toml b/pyproject.toml index cef84c3..3d800ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ +[tool.pytest.ini_options] +# bench/tests imports the bench package by name, so the repo root must be importable +# whichever way pytest is launched (pytest, or python -m pytest). +pythonpath = ["."] +testpaths = ["backend/tests", "bench/tests"] + [tool.mypy] python_executable = ".venv/bin/python" mypy_path = "."