From cf2de313dc9d0cd1d27e0eba65e963e1639eeedb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Fri, 9 Oct 2026 19:56:33 -0400 Subject: [PATCH] Correct the record the second review round measured against Round 2 found the functional side true and six record/discipline defects. - D9: the write-path line said "comparison uses secrets.compare_digest" as if it covered both tokens. It does not: the operator's plaintext is compared with compare_digest (api_auth.py:54, :110), a holder token is SQL equality on its sha256 digest (holder_store.verify_token) because no plaintext is stored. The boundary is now stated in verify_token's docstring as well as the record. - D10/D13: three historical numbers were written as if they were current. The four-file 59 is pinned to 2ea48d8 (test_trust_quarantine.py was 25 tests there, 26 from d56996b on), and both 401 gate lines are pinned to the commit they were measured on (83dfc14, 89c880f) with the current 406 named next to them. - D12: the compiler cache-key line overstated one direction. The old key had no identity part at all, so whoever compiled first got served to the next caller; an empty pack reaching the operator is one direction, not the mechanism. - D11: the p2-5 acceptance cell's suite count is pinned the same way. - D14: bench/video_audio_check.py wrote straight into bench/reports/, so a reviewer running it in its own worktree would have overwritten frozen evidence. It now takes --report/--payload and refuses a destination that already holds a frozen report; refuse_frozen_overwrite is tested, along with the guards that stop the harness from measuring nothing. Gates: pytest -q 410 passed; ruff clean; bench.filesize_check OK; bench.audit --product 33 harness files / 2 reports / 6 set files, clean; pyright 0 errors on the three touched files; mypy cold cache Success on the same three. --- CHECKLIST.md | 11 +- backend/src/ely_eye/holder_store.py | 6 +- bench/tests/test_video_audio_check.py | 56 ++++++ bench/video_audio_check.py | 270 ++++++++++++++++++++++++++ 4 files changed, 337 insertions(+), 6 deletions(-) create mode 100644 bench/tests/test_video_audio_check.py create mode 100644 bench/video_audio_check.py diff --git a/CHECKLIST.md b/CHECKLIST.md index 286217b..bfa7d55 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -369,7 +369,7 @@ - [x] p2-2.1 FastAPI 鉴权 + Origin 校验（`backend/src/ely_eye/api_auth.py`） - 事实：`require_write(request)` 挂在 **16** 个能改状态的路由上（`api.py` 8、`api_cartridges.py` 6、`api_holders.py` 2）。GET 和三个 POST（`/api/search`、`/api/who_knows`、`/api/ask`）不设卡。`/api/chat` 设卡，因为它写 cache 事件和 evidence 边。全部 19 个 POST/DELETE 里只有那 3 个没卡，逐个核过。 - 评审实测纠正了我的一句话：「查询不是写入」不成立——`/api/search` 与 `/api/ask` 会写 `term_df`（`text_index.py:134`，在 `_term_matches`（:121）里，rare-term 选择的 DF 缓存），只有 `/api/who_knows` 真的不写。根因不是给查询加卡（只读客户端就该能读），是那个缓存按**查询词**长大、上界由来人决定：现在 `matches == 0` 不写，所以行数上界是语料自己的词表，不是查询条数。测试 `test_a_word_the_library_lacks_leaves_nothing_behind`；评审另用真服务器量过：9 条只含生造词的 search/ask 请求零行增长，一条含真词的查询写 3 行（`"archive"* / "room"* / "thermostat"*`），40 词的生造查询仍不涨。 - - 语义（跑出来的）：Origin 不在白名单 → **403**；服务器没配 token → **403**「this server accepts no writes: start it with ELY_EYE_AUTH_TOKEN set」；缺头或错 token → **401**「write needs the token: send Authorization: Bearer $ELY_EYE_AUTH_TOKEN」。比对用 `secrets.compare_digest`，不是 `==`。 + - 语义（跑出来的）：Origin 不在白名单 → **403**；服务器没配 token → **403**「this server accepts no writes: start it with ELY_EYE_AUTH_TOKEN set」；缺头或错 token → **401**「write needs the token: send Authorization: Bearer $ELY_EYE_AUTH_TOKEN」。两条 token 走两条比对（评审第 2 轮 D9：这句原先一刀切写成「比对用 `compare_digest`」）：operator 的明文用 `secrets.compare_digest`（`api_auth.py:54`、`:110`）；holder token 库里只有 sha256 摘要，所以是 SQL 等值查摘要（`holder_store.verify_token` = `WHERE token_hash = ?`），没有明文可做常数时间比较。摘要等值不是常数时间，但明文从不落盘，这条边界现在写在 `verify_token` 的 docstring 里。 - 偏差（环境变量名是跑出来的）：本仓约定是 `env_prefix="ELY_EYE_"` 拼字段名，所以 `auth_token` 的环境变量是 **`ELY_EYE_AUTH_TOKEN`**，不是我起初写的 `ELY_EYE_TOKEN`。发现方式是 smoke 断言 401 却拿到 403，服务器把原因打印出来了。改名跨 7 个文件；树里现在 **13 行**出现这个名字（`api_auth.py` 2、`cli.py` 2、`test_api_auth.py` 1、`client.go` 1、`client_test.go` 1、`App.tsx` 1、`scripts/mcp_write_smoke.sh` 5）——我原先写「11 处」，后来又写成分项相加 11，都是错的，这行是评审逐文件 `grep -c` 数出来的。 - 客户端各面：Go `apiclient.Client.Token` 从 `ELY_EYE_AUTH_TOKEN` 读、每个请求带 `Authorization: Bearer`（`TestTheWriteTokenRidesOnEveryRequest` 用 `httptest` 验 POST 与 GET 都带、空 token 不带、环境值 trim）；前端把 token 存 `localStorage`（不进 bundle），顶栏一个 password 输入框；`ely-eye serve` 启动打一行说明写入要不要 token，token 本身不进日志。 - [x] p2-2.2 写入信任分级与隔离：未佐证不得被引用 @@ -398,7 +398,7 @@ - **评审第 4 轮 FAIL**（同一独立 pi，8 项全跑）：D11 / D13 / D14 的修法全部实测成立（它在 `9145a23` worktree 里逐字重现了门禁格那行数字：306 passed、bundle `index-u-5quhcj.js` 221.39 kB、`git show --name-only` 数出 23 个后端文件；验收格四文件与合跑 55 对上；点名的 7 个 `_ingest_*` 与两处 `trust.stamp` 行号逐个存在）。新缺陷四条：D15 就是上面那条混排 fail-open（真功能缺陷）；D16 「中英一样」实测不成立（拉丁词按长度吃掉三个名额，CJK 位置从不参与）；D17 第 2 轮记录行还写「我加测试后现在是 `8 + 17 + 8`」，与验收格自相矛盾（历史行必须钉提交或钉数字，不能写「现在」）；D18 我第 3 轮的编辑删掉了「可用词不足 3 个永不佐证」那条限制，行为与 docstring 一直在但记录没了、测试没人引用。评审自己跑的结果：`pytest -q` 320、ruff、`bench.filesize_check`、`bench.audit --product` 31 / 2 / 6 clean、pyright 0、mypy 冷缓存 24 / 9（同命令在 `9cb46c1` worktree 也是 24 / 9，文件集与逐条报错全等 ⇒ 净 0）、go 全套 + `gofmt -l` 空、`npm run build` 221.48 kB + eslint 0、smoke 4.861s PASS；把 `trust.py` 在两棵树之间互换，5 条新测试里 3 条在旧机制下真红（钉的是机制不是现状）；15 条中文/混排查询命中集与 `ff2471d` 逐字一致；纪律扫描干净（141 个 tracked `.py`、零多行整行注释块、最大 498 行、新名全部有引用、旧名零残留）。 - 评审量到一个**测量陷阱**（方法问题不是缺陷，但影响本仓所有「在某提交 worktree 里量的」断言）：`.venv/lib/python3.14/site-packages/__editable__.ely_eye-0.1.0.pth` 写的是主仓绝对路径，`pyproject.toml:4 pythonpath = ["."]` 只加仓根，所以在 worktree 里裸跑 pytest/python 导入的是**主树**的代码。任何「在 commit X 上量的」必须先 `PYTHONPATH=$PWD/backend/src`（它第一次换 `trust.py` 的实验因此三条对照假绿）。 - D15 修完的门禁：`pytest -q` **322 passed**（+2 条混排测试）、四文件合跑 **57 passed**、`ruff check` 干净、mypy 冷缓存 **24 errors / 9 files**（净 0）、`./node_modules/.bin/pyright` 对 `trust.py` 与 `test_trust_quarantine.py` **0 errors**、`bench/filesize_check` 干净、`bench.audit --product` 31 / 2 / 6 clean；本轮没动前端与 Go，`npm run build` 221.48 kB 与 go 全套是 `ff2471d` / `3340711` 上量的。 - - D20/D21 修完的门禁：`pytest -q` **324 passed**（+2 条填充词测试）、四文件合跑 **59 passed**、`ruff check` 干净、mypy 冷缓存 **24 errors / 9 files**（净 0）、`./node_modules/.bin/pyright` 对 `trust.py` 与 `test_trust_quarantine.py` **0 errors**、`bench/filesize_check` 干净、`bench.audit --product` 31 / 2 / 6 clean；前端与 Go 本轮未动，沿用 `8d84e03` 的 221.48 kB 与 go 全套。 + - D20/D21 修完的门禁：`pytest -q` **324 passed**（+2 条填充词测试）、四文件合跑 **59 passed**（`2ea48d8` 那轮的量——当时 `test_trust_quarantine.py` 是 25 条，`d56996b` 起才是 26 条；当前数字见上面验收格 60。评审第 2 轮 D10 判的就是这行没钉提交）、`ruff check` 干净、mypy 冷缓存 **24 errors / 9 files**（净 0）、`./node_modules/.bin/pyright` 对 `trust.py` 与 `test_trust_quarantine.py` **0 errors**、`bench/filesize_check` 干净、`bench.audit --product` 31 / 2 / 6 clean；前端与 Go 本轮未动，沿用 `8d84e03` 的 221.48 kB 与 go 全套。 - **评审第 5 轮 FAIL**（同一独立 pi，8 项全跑，D15–D18 的修法逐条跑验成立）：三项——D19 记录钉错 commit（见上）；D20 零成本填充词打开「短到没法比」的门（见上面 D20 那条，已修）；D21 披露不足：混排 claim 额外要求的那个拉丁词可能是虚词（实测 `the` / `ai`），于是混排规则真正的强度只在三个 CJK 位置，而一个把该拉丁词**翻译**成中文的合法 witness 会被判不一致（`…十八度 temperature` ← `…十八度 温度` → `False`）。修法是把虚词从被要求集合里剔掉（顺带修掉 D21 的副作用：`档案室的恒温器保持十八度 the` 现在被合法的中文复述 `恒温器一直放在档案室，读数十八度` 判 `True`，第 5 轮实测是 `False`），翻译边界保留为披露。评审同时实测确认：`8d84e03` 全套门禁与 `9cb46c1` 基线净 0；21 条混排查询在 `3340711` 与 `8d84e03` 之间 terms/表达式/命中集**完全一致**；把 `3340711` 的 `trust.py` 换进来时恰好 2 条新测试失败（证明它们钉的是机制）；两个 `third_party` 混排记忆不互证；p2-2 每一格数字复算一致。 - **评审第 6 轮 FAIL**（同一独立 pi，7 项全跑；功能面没找到 fail-open，D19/D20/D21 的修法全部实测成立，27 条中英混排查询在 `8d84e03` 与 `2ea48d8` 之间 terms/表达式/命中集逐字相同）：六条全部落在记录与纪律——D23 `:383` 三个钉全错（写 `trust.py:24` / 79 词 / 一行注释，实测 `:50` / 80 / 两行）；D24 把 `ai` 写成已剔除（`'ai' in STOPWORDS` 为假）；D25 `trust.py:48-49` 是 141 个 tracked `.py` 里唯一一处整行 `#` 连续 ≥2 的块（本仓库 p2-1 第 1 轮把多行注释判过缺陷）；D26 `:378` 又用「现在」描述机制且对 CJK 通道不成立；D27 虚词表把实义词当虚词这条限制未披露、无测试；D22 `scripts/mcp_write_smoke.sh` 就绪探测只看端口有没有人应答，撞车时测的是别人的服务器（实测 `FAIL: an untokened write returned HTTP 200, want 401`，并发双跑还出过 `cannot verify … model_hash` 与 `the source row vanished` 两条假失败）。修法：注释收一行、docstring 改写、`:378`/`:383` 按实测重写、补 D27 限制与测试、smoke 开跑前先探端口并检查自己进程存活。 - D22–D27 修完的门禁：`pytest -q` **325 passed**（+1 条 D27 测试）、四文件合跑 **60 passed**、`ruff check` 干净、mypy 冷缓存 **24 errors / 9 files**（净 0）、`./node_modules/.bin/pyright` 对 `trust.py` 与 `test_trust_quarantine.py` **0 errors**、`bench/filesize_check` 干净、`bench.audit --product` 31 / 2 / 6 clean、全仓整行 `#` 连续 ≥2 的块 **0**（tokenize 扫 141 个 tracked `.py`）。smoke 两条都实测：正常跑 `PASS: …` 且 exit 0；先用一台无关服务器占住 8799 再跑 → `FAIL: port 8799 already answers; stop that server or run with PORT=.` 且 **exit 1**（不再拿别人的服务器出结果）。前端与 Go 本轮未动，沿用 `8d84e03` 的 221.48 kB 与 go 全套。 @@ -434,11 +434,12 @@ - [x] p2-4.3 能力卡：让外部 agent 能路由进 ely-eye - `GET /api/card`（整库）与 `GET /api/card/{holder}`（一个人），CLI `ely-eye card`，MCP `ely_eye_card`（Go `apiclient.Card` / `HolderCard`，空 id 在**发请求之前**拒）。卡是**派生的不是声明的**（`card.build_card`）：`PRAGMA user_version`、`sources` 计数、`v_atoms_current` 按模态分组、每人 `(条数, MAX(ingested_at))`、每人模态/场景、合并目录、`has_audio = "audio" in senses`、挂载状态。所以它不能承诺一个 store 里没有的模态——`has_audio: false` 就是把视觉/音频腿的缺口在**别人提问之前**说出来。 - 无 token 就能读（smoke 实测 HTTP 与 MCP 两条路都通），且卡里**不含任何记忆正文**（smoke 断言 `secret not in json.dumps(card)`）；`token_issued` 只说「这个人有 token」，不发 token。卡的 schema 版本与整库卡一致这条也进了 smoke 的 tokenless 分支。 - - 偏差（都是跑出来的）：吊销过的 token 是**硬拒**（401「this token is neither the operator token nor a holder token」，`api_auth.py:59`），不降级成匿名；匿名调用者带 `?holder=`（空串）也是 403，不是「没声称」；能力卡除计数、分数、contact、声明过的话题之外还带每人的 `newest_memory` 时间戳与 `token_issued`，两者都不含记忆正文。`/api/memory/working` 与 `/api/memory/conflicts` 现在拒绝「谁也不是」的调用者（422，消息含「somebody」），因为这两件事本来就属于某个人；`compiler.py:41` 的缓存 key 加了眼睛——先前**根本没有身份成分**（`semantic_key(question, profile.value, self.db.revision())`；评审第 1 轮 D3：我写的「先前写 `as_holder or ""` 两边都哈希 `""`」那套机制在这个仓里从来没存在过），所以 operator 的上下文和陌生人的共用一个 key，陌生人那份空 pack 会端给 operator；现在 key 里带 `"the whole library"` 或 `as holder {as_holder!r}`，两种顺序实测四个 trace id 全不同（`None`→1 hit、`''`→0、`husband`→1、`nobody`→0）；`forget` 的级联、卡带成员、目录都不因 ACL 改动。 + - 偏差（都是跑出来的）：吊销过的 token 是**硬拒**（401「this token is neither the operator token nor a holder token」，`api_auth.py:59`），不降级成匿名；匿名调用者带 `?holder=`（空串）也是 403，不是「没声称」；能力卡除计数、分数、contact、声明过的话题之外还带每人的 `newest_memory` 时间戳与 `token_issued`，两者都不含记忆正文。`/api/memory/working` 与 `/api/memory/conflicts` 现在拒绝「谁也不是」的调用者（422，消息含「somebody」），因为这两件事本来就属于某个人；`compiler.py:41` 的缓存 key 加了眼睛——先前**根本没有身份成分**（`semantic_key(question, profile.value, self.db.revision())`；评审第 1 轮 D3：我写的「先前写 `as_holder or ""` 两边都哈希 `""`」那套机制在这个仓里从来没存在过），所以同一个问题在同一个 revision 上只有一个 key，**谁先编译谁的答案就被端给下一个人**——方向取决于谁先写，一份薄但非空的 pack 也会静默串用（评审第 2 轮 D12 实测：先编译陌生人那份，operator 随后拿到陌生人的 1 条命中；我原先写的「陌生人那份空 pack 会端给 operator」只是其中一种方向）；现在 key 里带 `"the whole library"` 或 `as holder {as_holder!r}`，两种顺序实测四个 trace id 全不同（`None`→1 hit、`''`→0、`husband`→1、`nobody`→0）；`forget` 的级联、卡带成员、目录都不因 ACL 改动。 - 验收：`pytest backend/tests/test_holder_acl.py backend/tests/test_visibility_rule.py backend/tests/test_transactive.py` → **13 + 2 + 19** 全绿（两 holder 互不可见、无 token 读不到私钥、`everyone` 无 token 可搜、`visible_to` 只放行被指名的人、声称别人 403 且点名两人、token 只出现一次且库里只有 sha256、吊销后旧 token 401、卡无 token 可读且不含正文、陌生人的图片 404 而无主的 200、`compile(as_holder="")` 返回 0 hits 且 `cache_trace_id` 与 `as_holder="wife"` / `None` 都不同）。跨进程：`bash scripts/mcp_write_smoke.sh` → 两行 PASS，第二行是「a caller with no token sees only public memory, a holder's token sees her own, naming somebody else is refused, and the capability card answers without a token」——它先证明无 token 搜不到私钥，再用 operator token 铸出妻子的 token、证明她能搜到自己的、搜不到 `local` 写的 `the oven needs descaling`，最后**把 MCP 进程不带 `ELY_EYE_AUTH_TOKEN` 重启一遍**，证明 Go 那条路守同一条规则（不是假设）。 - 顺手修的两条自己写的断言缺陷（都是跑出来的）：smoke 里 `if secret in found(None, token=token)` 是同义反复（搜的还是那句话，当然命中），改成拿**别人写的**记忆当探针；指纹探测 `GET /api/memory/working?holder=fingerprint` 现在返回 **403**（路由在、但匿名不能声称 holder），所以探测必须带 operator token——旧树上没有这条路由仍是 404，判别力保住。 - **评审第 1 轮 FAIL**（同一独立 pi，`209f2d0..83dfc14`，736 工具轮）：**功能面它全部实测成立**，记录面 8 条对不上，已全部按实测改掉——D1 「比对用 `compare_digest`」对 holder token 是假的（库里只有摘要，`verify_token` 是 SQL 等值，`compare_digest` 只用在 operator 的明文上）；D2 指针错（`register_minimal` 在 `memory.py:94`，不是 `:120`）；D3 「先前 `as_holder or ""` 两边哈希 `""`」描述的机制从来没存在过（旧 key 根本没有身份成分）；D4 `browse.py:14` 注释说「four — two here」而实际五份、这里三份（`:52`/`:66`/`:93`）；D5 指针被 as_of 改动挪了行（`vector_index.py:288`、`store_search.py:103`）；D6 我 prompt 里写「33 条路由」而实测父提交 34 个装饰器 / 31 条不同路径、HEAD 38 / 35；D7 semgrep 那条在它的环境里无法复现（`which semgrep` / `which opengrep` 都空），已改成「当时观察到的」并把防漂移的责任交给不依赖那个工具的测试；D8 「holder 也只能是 caller 自己…由 token 保证」与实测相反（只有 operator 能写，归属来自 `default_holder`，真正的保护是 `RememberRequest` 没有 holder 字段）。它同时实测确认：三条漏在父提交上是真的、在 HEAD 关上；38 条路由里会吐正文/字节的 7 条全带眼睛；11 个攻击全关（含吊销 token 401、holder token 不能写）；三态没被照瞎（CLI 命中、`bench.leg_check --probes 20` recall@10 0.900 / p50 72.3 ms、smoke PASS）；五份副本它自己数过，把 `? <> ''` 改掉立刻让 `test_visibility_rule.py` 红；缓存 key 两种顺序四个 trace id 全不同；smoke 新增那段是真判别子（父提交树会把私钥搜出来）。另采纳四条精度说明（吊销硬拒、匿名空串 403、能力卡还带时间戳与 `token_issued`、`leg_check` 的 commit 行是 cwd 的 HEAD）。 - - 门禁：`pytest -q` **401 passed**；`ruff check` 对 22 个改动文件干净；`bench/filesize_check` 干净；`bench.audit --product` → `audited 32 harness files, 2 reports, 6 set files` / `clean`；mypy 冷缓存 **24 errors in 9 files (checked 112 source files)**，与 `9cb46c1` 基线（24/9）**净新增 0**；`./node_modules/.bin/pyright` 对 17 个后端源文件 + 5 个测试文件 → **0 errors**；`go build` / `go vet` / `go test`（apiclient ok、mcpserver ok）+ `gofmt -l` 空；`npm run build` → `dist/assets/index-CSDHCFNo.js 221.48 kB`（bundle 哈希与 `246dae9` 相同，与「本轮没动前端」一致）+ eslint exit 0；`bash scripts/mcp_write_smoke.sh` 3.3s 两行 PASS。 + - **评审第 2 轮 FAIL**（同一独立 pi，`df812f3`，7 项全跑）：D1–D8 全部实测确认已修（它在 `209f2d0` worktree 里重现了旧 key 无身份成分、`git log -S` 重现了 `as_holder or ""` 只存在于 `83dfc14`）。as_of 的行为面它自己写脚本全测通：退役的记忆在 `as_of` 下应答、下周的事今天不应答、归档不挡 as_of、不跨 holder 借看、dense 腿与词腿读同一个时间窗（两条都只回 `drawer`）。`scripts/key_test.sh` 在 `df812f3` 上 exit 0（6 PASS + 1 NOT MEASURED），把 `store_search.py` 换回父提交就红在场景 3。新缺陷六条，五条记录一条纪律：D9 就是上面那条 `compare_digest` 一刀切；D10 是历史行没钉提交（已钉，并量出 `2ea48d8` 的 `test_trust_quarantine.py` 确实是 25 条、`d56996b` 起才是 26）；D11 p2-5 验收格写「全套 401」而当前 406；D12 缓存 key 那条的后果写过头（旧 key 无身份 ⇒ 谁先编译谁被端给下一个人，不是单向的空 pack）；D13 两条门禁格的 401 没钉提交（已分别钉 `83dfc14` / `89c880f` 并写明当前 406）；D14 `bench/video_audio_check.py` 写死 `bench/reports/`，在评审 worktree 里跑会脏了冻结报告（已加 `--scratch` / `--report` / `--payload`，默认值不变）。另采纳三条精度说明（不算缺陷）：吊销 token 的 401 消息只点名 operator 没点 holder、匿名带空 `?holder=` 是 403 而 smoke 断言写的是 401/403 并列、能力卡的 `token_issued` 会泄露「某人今天刚拿到 token」这种时间信号。 + - 门禁（`83dfc14` 那轮量的；as_of 之后当前是 406）：`pytest -q` **401 passed**；`ruff check` 对 22 个改动文件干净；`bench/filesize_check` 干净；`bench.audit --product` → `audited 32 harness files, 2 reports, 6 set files` / `clean`；mypy 冷缓存 **24 errors in 9 files (checked 112 source files)**，与 `9cb46c1` 基线（24/9）**净新增 0**；`./node_modules/.bin/pyright` 对 17 个后端源文件 + 5 个测试文件 → **0 errors**；`go build` / `go vet` / `go test`（apiclient ok、mcpserver ok）+ `gofmt -l` 空；`npm run build` → `dist/assets/index-CSDHCFNo.js 221.48 kB`（bundle 哈希与 `246dae9` 相同，与「本轮没动前端」一致）+ eslint exit 0；`bash scripts/mcp_write_smoke.sh` 3.3s 两行 PASS。 - [x] **p2-5** 文档单一真相 - [x] p2-5.1 `AGENTS.md` 重写并停止 gitignore - `.gitignore` 删了 `AGENTS.md` 那一行（`CLAUDE.md` 仍忽略，它不是本仓库的真相源），`git check-ignore AGENTS.md` 现在 exit 1。新文件 5.5KB，写的不是风格偏好而是**这个仓库付过学费的约束**：门禁命令清单（含 mypy 基线是 24 errors in 9 files、必须在分支点的 worktree 里量而不是猜）；一个 writer 一条规则（Go 不写 SQLite）；fail fast 必须点名（引真实报错 `nothing to forget: no atom or source was given`）；≤500 行、无死代码、注释单行、docstring 是契约；「缺席是一种状态」——`v_atoms_current` 隐藏而不删除。 @@ -455,7 +456,7 @@ - 诚实处理两处派生数字：`+0.454` 不在 `BASELINE_EXTERNAL.md` 里，是它两行（0.866 / 0.412）相减得来，Gaps 里写明「该报告只给两行与‘这段差值是 holder 边界的价值’这句话，差值是这里减出来的」；ASR 速度改成当前报告的数（258.18s 音频 / 3.12s，每视频 68.4–111.2×），不再引旧报告的 63.9–112.2。 - 偏差（顺手补的真缺口）：CLI `search` 只有 `--holder`（过滤器），**没有**办法用某个 holder 的眼睛看库——即无法回答「我的 agent 会看到什么」。加了 `--as-holder`（`SearchFilters.as_holder`）与两条断言（`--as-holder stranger` → `no_match`，`--as-holder wife` → 命中），否则三态眼睛只在 HTTP/MCP 两面成立而 CLI 面缺一个口。 - 验收：`grep -rn "MMMU-Pro=\|OCRBench=\|1-hop=\|2-hop=" *.md */README.md` → **0 命中**；README 里每个数字都能在报告里找到（0.863→LEGS、0.866/0.412→BASELINE_EXTERNAL、0.0717/0.2177→SCALE、165973/153857/106 per second→DENSE、0.242/533/68.4–111.2→ASR、模态行→LEGS、148.9GB→Gaps 附录 B）。唯一留下的模型分数是 `TRAINING_ARTIFACTS.md:84` 的 `citation accuracy 1.00`：它在证据块里（那是当天产出的工件原文），且紧跟的段落写明了它测的是什么、不测什么。 - - 门禁：`pytest -q` **401 passed**；`ruff check backend/src/ely_eye backend/tests bench` 干净；`bench.filesize_check` 干净；`bench.audit --product` → `audited 32 harness files, 2 reports, 6 set files` / `clean`；mypy 冷缓存 **24 errors in 9 files (checked 112 source files)** = `9cb46c1` 基线，净新增 0；`./node_modules/.bin/pyright backend/src/ely_eye/cli.py backend/tests/test_search_api.py` → **0 errors**；本轮未动 Go/前端/报告，所以 go/npm 套件沿用 `83dfc14` 的绿。 + - 门禁（`89c880f` 那轮量的；`eab6bd7` 加了 5 条 as_of 测试后当前是 406）：`pytest -q` **401 passed**；`ruff check backend/src/ely_eye backend/tests bench` 干净；`bench.filesize_check` 干净；`bench.audit --product` → `audited 32 harness files, 2 reports, 6 set files` / `clean`；mypy 冷缓存 **24 errors in 9 files (checked 112 source files)** = `9cb46c1` 基线，净新增 0；`./node_modules/.bin/pyright backend/src/ely_eye/cli.py backend/tests/test_search_api.py` → **0 errors**；本轮未动 Go/前端/报告，所以 go/npm 套件沿用 `83dfc14` 的绿。 --- diff --git a/backend/src/ely_eye/holder_store.py b/backend/src/ely_eye/holder_store.py index db4bd49..6d5e88d 100644 --- a/backend/src/ely_eye/holder_store.py +++ b/backend/src/ely_eye/holder_store.py @@ -121,7 +121,11 @@ class HolderStore: return entry def verify_token(self, token: str) -> str | None: - """Which holder a bearer token speaks for, or None when it speaks for nobody.""" + """Which holder a bearer token speaks for, or None when it speaks for nobody. + + The store keeps only the sha256 digest, so this is SQL equality on the digest: no plaintext + exists here to compare in constant time, and a copied database cannot replay a token. + """ if not token: return None with self.database.connect() as connection: diff --git a/bench/tests/test_video_audio_check.py b/bench/tests/test_video_audio_check.py new file mode 100644 index 0000000..6bac428 --- /dev/null +++ b/bench/tests/test_video_audio_check.py @@ -0,0 +1,56 @@ +"""The video+audio harness must refuse to measure nothing, and must not touch frozen evidence.""" + +from pathlib import Path +from typing import Any + +import pytest + +from bench import video_audio_check + + +def test_a_frozen_report_is_not_overwritten_by_a_harness( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A reviewer runs harnesses in a worktree of its own; the committed reports stay put.""" + monkeypatch.setattr(video_audio_check, "PAYLOAD", tmp_path / "video_audio.json") + frozen = tmp_path / "VIDEO_AUDIO.md" + frozen.write_text("committed evidence", encoding="utf-8") + with pytest.raises(SystemExit, match="is a frozen report"): + video_audio_check.refuse_frozen_overwrite(frozen) + elsewhere = tmp_path / "scratch" / "VIDEO_AUDIO.md" + elsewhere.parent.mkdir() + elsewhere.write_text("scratch", encoding="utf-8") + video_audio_check.refuse_frozen_overwrite(elsewhere) + + +def test_a_missing_speech_snapshot_says_so_before_ranking(tmp_path: Path) -> None: + """Without a snapshot every video defers, and the after row would equal the before row.""" + with pytest.raises(SystemExit, match="no speech snapshot"): + video_audio_check.add_speech(tmp_path, [tmp_path / "a.mp4"], tmp_path / "no-snapshot", 0) + + +def test_a_store_without_video_sources_refuses(tmp_path: Path) -> None: + """Speech has to join to a video source; a store with none would measure a new corpus.""" + import sqlite3 + + videos = tmp_path / "videomme" + videos.mkdir() + (videos / "a.mp4").write_bytes(b"\x00") + with pytest.raises(SystemExit, match="are not here"): + video_audio_check.guard_videos(tmp_path / "nowhere", tmp_path) + store = tmp_path / "store" + (store / "data").mkdir(parents=True) + connection = sqlite3.connect(store / "data" / "ely_eye.sqlite") + connection.execute("CREATE TABLE sources (source_id text, kind text)") + connection.commit() + connection.close() + with pytest.raises(SystemExit, match="holds no video sources"): + video_audio_check.guard_videos(videos, store) + + +def test_every_measured_video_query_carries_gold() -> None: + """An answerable-only set: a query with no gold would score 0 against nothing.""" + queries: list[Any] = video_audio_check.video_queries() + assert queries, "the frozen suite must carry answerable video queries" + assert all(query.modality == video_audio_check.VIDEO_MODALITY for query in queries) + assert all(query.gold for query in queries) diff --git a/bench/video_audio_check.py b/bench/video_audio_check.py new file mode 100644 index 0000000..4a40d8d --- /dev/null +++ b/bench/video_audio_check.py @@ -0,0 +1,270 @@ +"""What speech is worth on the video questions: the same store and probes, with audio added. + +The frozen stores were ingested before ASR landed, so every video answers from frames and OCR +alone and video recall@10 sits at 0.455 while code sits at 0.945. This harness copies the dense +store, re-ingests the same Video-MME files with a speech snapshot wired in, and ranks the same +probes before and after. The only thing that moved between the two rows is the speech. + +Run: python -m bench.video_audio_check +Writes: bench/reports/VIDEO_AUDIO.md, bench/reports/video_audio.json +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sys +import time +from pathlib import Path +from typing import Any + +from bench.leg_check import SHIPPED_TOP_K, _groups, _source_ids, rank_all +from bench.metrics import aggregate, score_query +from bench.query_set import load_queries, scale_probes, source_times +from bench.report import git_commit +from bench.run import latency +from bench.scratch import copy_store +from bench.sets_common import DATA_ROOT +from bench.systems import machine_facts + +DEFAULT_HOME = Path("bench/runs/dense_scale") +DEFAULT_SCRATCH = Path("bench/runs/video_audio") +DEFAULT_ASR = Path("bench/models/faster-whisper-tiny") +DEFAULT_PROBES = 300 +VIDEO_MODALITY = "video" +REPORT = Path("bench/reports/VIDEO_AUDIO.md") +PAYLOAD = Path("bench/reports/video_audio.json") + + +def video_queries() -> list[Any]: + """Every answerable video question in the frozen suite, measured whole rather than sampled.""" + queries = [query for query in load_queries() if query.modality == VIDEO_MODALITY and query.gold] + if not queries: + raise SystemExit("the frozen suite carries no answerable video query; there would be nothing to score") + return queries + + +def guard_videos(videos: Path, store: Path) -> list[Path]: + """The files the store already holds, so re-ingesting adds speech instead of a second copy.""" + import sqlite3 + + if not videos.is_dir(): + raise SystemExit( + f"{videos}: the Video-MME files are not here; they are staged from the corpus zips at " + "the data root (set ELY_EYE_BENCH_DATA to wherever they live)" + ) + files = sorted(videos.glob("*.mp4")) + if not files: + raise SystemExit(f"{videos}: no .mp4 files to re-ingest") + with sqlite3.connect(f"file:{store / 'data' / 'ely_eye.sqlite'}?mode=ro", uri=True) as connection: + stored = {row[0] for row in connection.execute("SELECT source_id FROM sources WHERE kind = 'video'")} + if not stored: + raise SystemExit(f"{store}: the store holds no video sources, so speech would have nothing to join to") + return files + + +def speech_facts(store: Path) -> dict[str, Any]: + """How much speech the store carries, and how many words of it.""" + import sqlite3 + + with sqlite3.connect(f"file:{store / 'data' / 'ely_eye.sqlite'}?mode=ro", uri=True) as connection: + atoms, characters = connection.execute( + "SELECT count(*), coalesce(sum(length(text)), 0) FROM atoms WHERE modality = 'audio'" + ).fetchone() + return {"atoms": int(atoms), "characters": int(characters)} + + +def rank_arm(home: Path, probes: list[Any], top_k: int) -> dict[str, Any]: + """Rank the probes through the shipped words-only path and score them.""" + from ely_eye.config import Settings + from ely_eye.db import Database + from ely_eye.retrieval import RetrievalService + + settings = Settings( + home=home, + dense_embeddings_enabled=False, + embedding_model_dir=None, + clip_model_dir=None, + ) + if settings.retrieval_top_k != SHIPPED_TOP_K: + raise SystemExit( + f"the product asks for {settings.retrieval_top_k} atoms per search but this harness " + f"labels its row 'as shipped' at {SHIPPED_TOP_K}; update the label before trusting it" + ) + database = Database(settings) + service = RetrievalService(settings=settings, database=database) + ids = _source_ids(database) + rankings, distinct, timings = rank_all(service, ids, probes, top_k) + scores = [ + score_query(query.query_id, ranking, set(query.gold), query.holder) + for query, ranking in zip(probes, rankings, strict=True) + ] + summary = aggregate(scores) + timing = latency(timings) + return { + "queries": len(probes), + "top_k": top_k, + "recall_at_10": summary["recall_at_10"], + "mrr": summary["mrr"], + "hit_at_10": summary["hit_at_10"], + "wilson_lower": summary["hit_at_10_wilson_lower"], + "p50_seconds": timing["p50"], + "p95_seconds": timing["p95"], + "median_distinct_sources": statistics.median(distinct), + **_groups(probes, scores, rankings, source_times()), + } + + +def add_speech(home: Path, files: list[Path], asr: Path, limit: int) -> dict[str, Any]: + """Re-ingest the same files with the speech snapshot, and time the whole pass.""" + from ely_eye.config import Settings + from ely_eye.ingestion import IngestionService + + if not (asr / "model.bin").exists(): + raise SystemExit( + f"{asr}: no speech snapshot here; the ingest would defer every video and the after row " + "would silently equal the before row" + ) + if limit: + files = files[:limit] + settings = Settings(home=home, asr_model_dir=asr) + service = IngestionService(settings) + began = time.perf_counter() + batches = 0 + for index, path in enumerate(files, start=1): + service.ingest_path(path) + batches += 1 + if index % 10 == 0: + print(f" ingested {index}/{len(files)}", flush=True) + return {"files": batches, "seconds": time.perf_counter() - began} + + +def measure(home: Path, scratch: Path, videos: Path, asr: Path, probes: int, limit: int) -> dict[str, Any]: + """Score the video questions and the standard probe sample before and after the speech pass.""" + if not (home / "data" / "ely_eye.sqlite").exists(): + raise SystemExit(f"{home}: no store to copy; run `python -m bench.dense_scale` first") + files = guard_videos(videos, home) + whole = video_queries() if not limit else video_queries()[:limit] + sample = [query for query in scale_probes(load_queries(), probes) if query.gold] + if limit: + sample = [query for query in sample if query.modality == VIDEO_MODALITY][:limit] + print(f"{len(whole)} answerable video queries, {len(sample)} stratified probes, {len(files)} files", flush=True) + + copy_home, copied = copy_store(home, scratch) + print(f"copied the store in {copied:.1f}s to {copy_home}", flush=True) + before_speech = speech_facts(copy_home) + if before_speech["atoms"]: + print(f"note: the copy already carries {before_speech['atoms']} speech atoms", flush=True) + before_video = rank_arm(copy_home, whole, SHIPPED_TOP_K) + before_sample = rank_arm(copy_home, sample, SHIPPED_TOP_K) + print( + f" before: video recall@10 {before_video['recall_at_10']:.3f}, " + f"probes {before_sample['recall_at_10']:.3f}", + flush=True, + ) + + ingest = add_speech(copy_home, files, asr, limit) + after_speech = speech_facts(copy_home) + if after_speech["atoms"] == before_speech["atoms"]: + raise SystemExit( + f"re-ingesting {len(files)} files added no speech atoms; the snapshot at {asr} is " + "available but produced nothing, so the after row would be a measurement of nothing" + ) + after_video = rank_arm(copy_home, whole, SHIPPED_TOP_K) + after_sample = rank_arm(copy_home, sample, SHIPPED_TOP_K) + print( + f" after: video recall@10 {after_video['recall_at_10']:.3f}, " + f"probes {after_sample['recall_at_10']:.3f}, speech atoms {after_speech['atoms']}", + flush=True, + ) + return { + "commit": git_commit(), + "machine": machine_facts(), + "source_store": str(home), + "videos": str(videos), + "files": len(files), + "asr_model": str(asr), + "ingest": ingest, + "speech_before": before_speech, + "speech_after": after_speech, + "before": {"video": before_video, "probes": before_sample}, + "after": {"video": after_video, "probes": after_sample}, + } + + +def write_markdown(report: dict[str, Any], destination: Path) -> None: + """Print the two rows beside each other, with the load they were measured under.""" + load = report["machine"].get("load_average_1_5_15") + lines = [ + "# What speech is worth on the video questions", + "", + f"- commit: `{report['commit']}`", + f"- store measured: a copy of `{report['source_store']}`; the source is never opened for write", + f"- machine: {report['machine'].get('cpus')} cpus, load {load}", + f"- speech snapshot: `{report['asr_model']}` (the shipped default; its word error rate is in ASR.md)", + f"- re-ingest: {report['ingest']['files']} files in {report['ingest']['seconds']:.0f}s", + ( + f"- speech atoms: {report['speech_before']['atoms']} before → {report['speech_after']['atoms']} after" + f" ({report['speech_after']['characters']} characters)" + ), + "", + "The only difference between the rows is the speech: same store, same files, same probes, same words-only path.", + "", + "| asked | n | recall@10 | MRR | hit@10 | 95% lower | p50 s |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for stage in ("before", "after"): + for arm, label in (("video", "every video question"), ("probes", "the stratified probe sample")): + row = report[stage][arm] + lines.append( + f"| {label} ({stage}) | {row['queries']} | {row['recall_at_10']:.3f} | {row['mrr']:.3f} | " + f"{row['hit_at_10']:.3f} | {row['wilson_lower']:.3f} | {row['p50_seconds']:.3f} |" + ) + gained = report["after"]["video"]["recall_at_10"] - report["before"]["video"]["recall_at_10"] + overall = report["after"]["probes"]["recall_at_10"] - report["before"]["probes"]["recall_at_10"] + lines += [ + "", + f"Speech moved video recall@10 by {gained:+.3f} and the whole probe sample by {overall:+.3f}.", + "", + "Latency is read from the same pass and moves with machine load; the recall columns do not.", + "", + ] + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text("\n".join(lines), encoding="utf-8") + + +def refuse_frozen_overwrite(destination: Path) -> None: + """A reviewer running this in a worktree of its own must not overwrite the committed evidence.""" + if destination.parent == PAYLOAD.parent and destination.exists(): + raise SystemExit( + f"{destination} is a frozen report; point --report and --payload somewhere else, or " + "delete the frozen copy on purpose rather than under a harness" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--home", type=Path, default=DEFAULT_HOME, help="store to copy") + parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH, help="where the copy lives") + parser.add_argument("--videos", type=Path, default=DATA_ROOT / "videomme", help="where the mp4s are staged") + parser.add_argument("--asr", type=Path, default=DEFAULT_ASR, help="speech snapshot") + parser.add_argument("--probes", type=int, default=DEFAULT_PROBES, help="probe sample size") + parser.add_argument("--limit", type=int, default=0, help="smoke knob: only this many files and queries") + parser.add_argument("--report", type=Path, default=REPORT, help="markdown destination") + parser.add_argument("--payload", type=Path, default=PAYLOAD, help="json destination") + arguments = parser.parse_args(argv) + for destination in (arguments.report, arguments.payload): + refuse_frozen_overwrite(destination) + report = measure( + arguments.home, arguments.scratch, arguments.videos, arguments.asr, arguments.probes, arguments.limit + ) + arguments.payload.parent.mkdir(parents=True, exist_ok=True) + arguments.payload.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + write_markdown(report, arguments.report) + print(f"wrote {arguments.report}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main())