From 2fa91abbc5d7b5f97523909eb677dc5b2dfce08d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 7 Oct 2026 22:39:27 -0400 Subject: [PATCH] Freeze the speech report against the commit that fixes the prose The report is regenerated after 10a39c5 so its stamp names the code it describes. Speed is quoted as a range in the checklist because transcribe seconds move with machine load (3.26s then 3.12s on this box) while WER, denominators and word counts do not move at all. Numbers: 258.18s of audio, WER 0.242 over 533 reference words, 80 audio atoms and 563 word timestamps read back from the store, 0 out of order and 0 before their own segment measured on the stored words. --- CHECKLIST.md | 11 +++++++++-- bench/reports/ASR.md | 16 ++++++++-------- bench/reports/asr.json | 26 ++++++++++++++------------ 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/CHECKLIST.md b/CHECKLIST.md index d85c25d..94aa412 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -159,8 +159,9 @@ - 验收：`pytest backend/tests/test_asr.py` 绿（13 个测试）；ASR **实跑**，不是 DEFERRED - 实测证据（`bench/reports/ASR.md`，`.venv/bin/python -m bench.asr_check` 生成）：3 段 Video-MME 真视频 **258.18 秒**音频（容器时长 88.93 / 109.18 / 60.07），对着第三方字幕 WER **0.242**——分母是 - **533 个参考词**（听到 563 个词），三行各自的参考词数是 201 / 296 / 36。CPU int8 4 线程 - 3.26 秒转完，逐行 63.9× / 82.4× / 112.2× 实时。走产品路径摄入后从库里读回：**80 个 audio atom**、 + **533 个参考词**（听到 563 个词），三行各自的参考词数是 201 / 296 / 36。CPU int8 4 线程跑完这 258 秒 + 音频：两份已提交报告分别 3.26 秒与 3.12 秒（逐行 63.9–112.2× 与 68.4–111.2× 实时——转写秒数随 + 机器负载浮动，WER 与词数不随负载动）。走产品路径摄入后从库里读回：**80 个 audio atom**、 **563 个词级时间戳**、0 个时序倒错、0 个早于自己所属片段（总共 133 个 atom）。模型 `Systran/faster-whisper-tiny`（78.2MB，model.bin sha256 dcb76c6586fc06cb…），远低于 500MB 上限。 - 可复现性是量出来的：解码参数钉死 `beam_size=1, temperature=(0.0,), condition_on_previous_text=False, @@ -169,6 +170,12 @@ 直接 `TypeError`）。同一台机器连跑两遍：WER 0.242 → 0.242，转写文本逐字节相同。 - 字幕不是逐字稿，所以这个 WER **不是任何方向的界**：字幕漏词也会掩盖模型漏词。 报告把分母写在表里，不写「上界」这种断言。 + - review 第二轮 FAIL 的 4 条全是数字与措辞，也都改了：docstring 里的 1.7 秒是钉参数前那次运行的残留 + （现在不写数字，只指向报告；并且不再把容器时长叫「语音时长」，因为 `vad_filter=False`）； + CHECKLIST 里的 3.55 秒 / 61.6× 与「31 段语音」同样是旧运行残留（钉死后是 38/39/3 段，加起来 + 正好是库里的 80 个 atom）；报告那句「库里有 80 个 atom、0 个时序倒错」原本把只在模型输出上 + 算过的不变量说成库侧的，现在 harness 对库里存的 words 重算（`out_of_order_in_store` / + `before_their_segment_in_store`）。速度只报区间，因为转写秒数跟机器负载走。 - 偏差与根因： - 验收路径 `tests/test_audio_ingest.py` 不存在，实际是 `backend/tests/test_asr.py`。 - `Modality` 只加 `audio`，没加 `transcript`：一份转写就是带时间戳的 audio atom， diff --git a/bench/reports/ASR.md b/bench/reports/ASR.md index 8fa8cc9..0ce3931 100644 --- a/bench/reports/ASR.md +++ b/bench/reports/ASR.md @@ -1,21 +1,21 @@ -# ASR: what the speech reader hears (commit 709104e80) +# ASR: what the speech reader hears (commit 10a39c512) -- generated 2026-10-08T02:24:47+00:00 +- generated 2026-10-08T02:37:10+00:00 - model: Systran/faster-whisper-tiny (cpu int8, 4 threads), model.bin sha256 dcb76c6586fc06cb… - decode: word_timestamps=True, beam_size=1, temperature=(0.0,), condition_on_previous_text=False, vad_filter=False - reference: Video-MME community subtitles: third-party captions, not verbatim transcripts -- machine: {'system': 'Linux', 'release': '6.18.40.1-microsoft-standard-WSL2', 'cpus': 32, 'threads_per_query': 4, 'load_average_1_5_15': [9.65, 2.54, 0.88]} +- machine: {'system': 'Linux', 'release': '6.18.40.1-microsoft-standard-WSL2', 'cpus': 32, 'threads_per_query': 4, 'load_average_1_5_15': [7.73, 2.66, 1.58]} | video | audio s | transcribe s | ×realtime | reference words | words | WER | mean word confidence | | --- | --- | --- | --- | --- | --- | --- | --- | -| 0IdYJGBmguM | 88.93 | 1.39 | 63.9 | 201 | 219 | 0.229 | 0.861 | -| 1q-5IIyZL20 | 109.18 | 1.33 | 82.4 | 296 | 313 | 0.253 | 0.821 | -| 1tidNDIpKSY | 60.07 | 0.54 | 112.2 | 36 | 31 | 0.222 | 0.852 | +| 0IdYJGBmguM | 88.93 | 1.3 | 68.4 | 201 | 219 | 0.229 | 0.861 | +| 1q-5IIyZL20 | 109.18 | 1.28 | 85.4 | 296 | 313 | 0.253 | 0.821 | +| 1tidNDIpKSY | 60.07 | 0.54 | 111.2 | 36 | 31 | 0.222 | 0.852 | -Over 3 videos and 258.18 seconds of audio: word error rate **0.242** over 533 reference words (563 words heard), transcribed in 3.26 seconds. +Over 3 videos and 258.18 seconds of audio: word error rate **0.242** over 533 reference words (563 words heard), transcribed in 3.12 seconds. Repeating the pass on the same machine gave WER 0.242 then 0.242; the transcripts were byte-identical. Decoding is greedy with no retry ladder, so a rerun is the same answer, not a new draw. -Ingested through the product path, the store holds 80 audio atoms from 3 sources carrying 563 word timestamps (0 out of order, 0 before their own segment) out of 133 atoms in total. +Ingested through the product path, the store holds 80 audio atoms from 3 sources carrying 563 word timestamps (0 out of order, 0 before their own segment) out of 133 atoms in total, checked on the words as stored rather than on the model's answer. The captions are not a transcript, so this WER is not a bound in either direction: a caption that drops words also hides words the model dropped. The third row has only 36 reference words, where one word moves its rate by several points — read the denominators, not the decimals. diff --git a/bench/reports/asr.json b/bench/reports/asr.json index 74fd62e..5b70bfb 100644 --- a/bench/reports/asr.json +++ b/bench/reports/asr.json @@ -1,6 +1,6 @@ { - "generated_utc": "2026-10-08T02:24:47+00:00", - "commit": "709104e80915e43a1095a270a87f09cbb611d51b", + "generated_utc": "2026-10-08T02:37:10+00:00", + "commit": "10a39c5125ebaf73e0ff7174b3af7af4c32bcf05", "model": { "id": "Systran/faster-whisper-tiny", "sha256_model_bin": "dcb76c6586fc06cbdac6dd21f14cfd129cc4cdd9dce19bf4ffa62e59cbe6e6d1", @@ -21,9 +21,9 @@ "cpus": 32, "threads_per_query": 4, "load_average_1_5_15": [ - 9.65, - 2.54, - 0.88 + 7.73, + 2.66, + 1.58 ] }, "passes": { @@ -54,7 +54,7 @@ "totals": { "videos": 3, "audio_seconds": 258.18, - "transcribe_seconds": 3.26, + "transcribe_seconds": 3.12, "word_error_rate": 0.242, "reference_words": 533, "hypothesis_words": 563, @@ -64,14 +64,16 @@ "atoms_all": 133, "speech_atoms": 80, "word_timestamps": 563, - "sources": 3 + "sources": 3, + "out_of_order_in_store": 0, + "before_their_segment_in_store": 0 }, "videos": [ { "video": "0IdYJGBmguM", "seconds": 88.93, - "transcribe_seconds": 1.39, - "real_time_factor": 63.9, + "transcribe_seconds": 1.3, + "real_time_factor": 68.4, "segments": 38, "reference_words": 201, "hypothesis_words": 219, @@ -85,8 +87,8 @@ { "video": "1q-5IIyZL20", "seconds": 109.18, - "transcribe_seconds": 1.33, - "real_time_factor": 82.4, + "transcribe_seconds": 1.28, + "real_time_factor": 85.4, "segments": 39, "reference_words": 296, "hypothesis_words": 313, @@ -101,7 +103,7 @@ "video": "1tidNDIpKSY", "seconds": 60.07, "transcribe_seconds": 0.54, - "real_time_factor": 112.2, + "real_time_factor": 111.2, "segments": 3, "reference_words": 36, "hypothesis_words": 31,