diff --git a/CHECKLIST.md b/CHECKLIST.md index 02ab60e..7cea1a8 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -105,12 +105,25 @@ - 根因修复：`_answered()` 里 `readable = owner in (as_holder, LIBRARY)` 是 SQL 可见性子句之外的第二份、更严的规则，与 `visible_to=[husband]` 冲突（共享记忆被错误隐藏）→ 删掉这份重复判断，SQL 的 `as_holder` 可见性子句是唯一权威。 - 偏差：Go 侧 `limit`/`max_hops` 零值曾当真实数字发出，被 API 以 `greater_than` 拒 → 未设置的可选字段一律省略，由服务端定默认值。 - 验收：`pytest tests/test_transactive.py` 绿；钥匙测试场景通过 -- [ ] **p0-6** 证明套件去自证 - - [ ] p0-6.1 删 arena 答案入墨（`evals.py:1408-1409` 连接线、`evals.py:1417-1426` 白圈） - - [ ] p0-6.2 删 best-of-N 重摇与达标即停 - - [ ] p0-6.3 删自造矛盾场景（出题者≠判分者） - - [ ] p0-6.4 删缓存/证据图自触发式检查 - - [ ] p0-6.5 模型相关检查标 `DEFERRED`，不进 pass +- [x] **p0-6** 证明套件去自证 + - [x] p0-6.1 删 arena 答案入墨（`evals.py:1408-1409` 连接线、`evals.py:1417-1426` 白圈） + - 偏差：入墨是唯一的可读信号——`render_visual_hash` 只画噪声、不画字，答案槽被青色线+白圈标出后，模型不需要检索也能答。所以删墨的同时把 id 印上瓦片（`proofs/arena.py:render_visual_hash` 白底黑字、每行 8 字符、字号 `tile_size//8`），否则删墨 = 删题。 + - [x] p0-6.2 删 best-of-N 重摇与达标即停 + - `collect_model_visual_proofs(evaluator, hops, token_equivalent, attempts)` 固定跑 `HASHHOP_ATTEMPTS = 8` 题；旧代码 `while len(proofs) < max_samples` 里一达标就 break（`evals.py:2365`），等于弱模型反复重摇到及格。 + - [x] p0-6.3 删自造矛盾场景（出题者≠判分者） + - 偏差：自造场景没有删掉（删了就没得测），而是从代码搬进**数据**：`proofs/contradiction_scenarios.json`（12 场景，sha256 `5f93aba09cbfc32f…`），`corpus_path(settings.data_dir)` 优先读运维者放的第三方集，`ely-eye contradiction-corpus` 写默认集。检查项**永久 DEFERRED**，直到磁盘上不是自带的那份；reason 里写明「生产者用自家例题判分不算证据」。 + - [x] p0-6.4 删缓存/证据图自触发式检查 + - 缓存：旧检查只要 `any(layer.entries > 0)` 就算过（`evals.py:1252-1265`）。现在拿 store 里真实的一条 atom 拼一个带 nonce 的问题，编译两次，读 `cache_events` 的 miss→hit 计数；空 store 直接 failed（空上下文会让两次编译「一致」得毫无意义）。顺带修掉真实双计数：`read_semantic_cache` 与 `compiler.compile` 各记一次事件，现在读是纯读、编译恰好记一次（缓存命中带 hits 才记 hit，空缓存记 miss 并附 reason）。 + - 仓库覆盖：旧检查 `indexed_file_count > 0` + 有调用图 + 有 git 历史就算过，不碰 store。现在必须 `sources > 0 && atoms > 0`，否则 failed 并说明「仓库上下文建了但没进 store」。 + - 研究模块：旧检查断言八个模块各返回非空列表，而那些向量是 SHA256 派生的假向量——**不可能失败**。改为 `status="removed"`，artifact 记下为什么撤下、重新加回的条件。 + - [x] p0-6.5 模型相关检查标 `DEFERRED`，不进 pass + - `ProofCheck.status` 加 `deferred`/`removed` + `reason` + `rerun`；`_deferred_without_model()` 在 `model_dir` 缺失时产出 deferred 并点名缺什么。聚合改成 `failed > deferred > passed`：只要还有没跑成的主张，整个套件就不说「通过」。每条 failed/deferred 都带 why 与可执行的 rerun 命令（`proofs/table.py` 渲染）。 + - [x] p0-6.6 顺手还清的文件尺寸债 + - `evals.py` 2427 行 → `proofs/` 包 15 个模块，最大 445 行（`checks_cartridge.py`），全部 ≤500 行；`suite.py` 218 行只剩「跑哪些检查、怎么定论」。`proofs/__main__.py` 删掉（它的 `latest_report()` 与 `PRDProofSuite.latest()` 是两份 glob 不同的重复实现），渲染移到 `proofs/table.py`，`latest-proof-suite` 成为唯一入口并在 failed 时 `exit 1`。 + - 顺带修掉两个被证明测试逼出来的真 bug： + 1. `text_index.fts_query` 把拉丁词用空格连起来 = FTS5 隐式 AND，长问题永远查不到短答案（实测「Which Ely-Eye PRD requirements prove long context memory, HashHop, and adapters?」→ 0 命中）。改成 OR + bm25 rank 排序。 + 2. `profiles.windows_path_to_wsl()` 对本来就是 POSIX 的路径也拼 `/mnt/` + 空 drive，产出 `/mnt//home/...`，六个 runtime profile 的探针全部 `probe_exit_code: 1`。 + - 验收：`python -m pytest backend/tests/test_proofs.py -q` → 12 passed；`ELY_EYE_HOME=/tmp/p06home ely-eye proof-suite --cartridge-id p06_20cf30f7804d && ely-eye latest-proof-suite` → 15 行判决（5 failed / 3 passed / 6 deferred / 1 removed），每条带 why 与 rerun。 - 验收：`python -m ely_eye.proofs --report` 逐条 PASS/DEFERRED/REMOVED + 理由 --- diff --git a/backend/src/ely_eye/api.py b/backend/src/ely_eye/api.py index 36560a6..582ea90 100644 --- a/backend/src/ely_eye/api.py +++ b/backend/src/ely_eye/api.py @@ -24,7 +24,6 @@ from .compiler import ContextCompiler from .config import Settings, get_settings from .db import Database from .embeddings import EmbeddingService -from .evals import HashHopEvaluator, PRDProofSuite from .evaluation_schemas import ( BenchmarkPredictionReport, BenchmarkRegistryReport, @@ -38,6 +37,7 @@ from .evidence_graph import EvidenceGraphService from .ingestion import IngestionService from .memory import MemoryService from .profiles import runtime_profile_report +from .proofs import HashHopEvaluator, PRDProofSuite from .research_modules import ResearchModuleService from .retrieval import RetrievalService from .runtime import QwenRuntime, RuntimeUnavailableError diff --git a/backend/src/ely_eye/cache_fabric.py b/backend/src/ely_eye/cache_fabric.py index 0ffeac7..078b611 100644 --- a/backend/src/ely_eye/cache_fabric.py +++ b/backend/src/ely_eye/cache_fabric.py @@ -9,6 +9,8 @@ from .config import Settings, get_settings from .db import Database from .schemas import CacheLayerStatus +SEMANTIC_LAYER = "semantic" + class CacheFabric: def __init__(self, settings: Settings | None = None, database: Database | None = None) -> None: @@ -16,7 +18,7 @@ class CacheFabric: self.db = database or Database(self.settings) def semantic_key(self, question: str, profile: str) -> str: - return hashlib.sha256(f"compiler-v2\n{profile}\n{question}".encode("utf-8")).hexdigest() + return hashlib.sha256(f"compiler-v2\n{profile}\n{question}".encode()).hexdigest() def record_hit(self, layer: str, cache_key: str, bytes_estimate: int = 0, metadata: dict[str, Any] | None = None) -> None: self.db.log_cache_event(layer, cache_key, "hit", bytes_estimate, metadata) @@ -29,15 +31,17 @@ class CacheFabric: path.parent.mkdir(parents=True, exist_ok=True) data = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") path.write_bytes(data) - self.db.log_cache_event("semantic", cache_key, "write", len(data), {"path": str(path)}) + self.db.log_cache_event(SEMANTIC_LAYER, cache_key, "write", len(data), {"path": str(path)}) def read_semantic_cache(self, cache_key: str) -> dict[str, Any] | None: + """Return the cached payload, or None. The caller traces the event: one compile, one event.""" path = self.settings.cache_dir / "semantic" / f"{cache_key}.json" if not path.exists(): - self.record_miss("semantic", cache_key) return None - self.record_hit("semantic", cache_key, path.stat().st_size) - return json.loads(path.read_text(encoding="utf-8")) + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{path}: the semantic cache entry is unreadable ({error})") from error def statuses(self) -> list[CacheLayerStatus]: rows = self.db.cache_layer_rows() diff --git a/backend/src/ely_eye/cli.py b/backend/src/ely_eye/cli.py index c880fef..09f2bfb 100644 --- a/backend/src/ely_eye/cli.py +++ b/backend/src/ely_eye/cli.py @@ -20,10 +20,12 @@ from .cli_transactive import register as register_transactive_commands from .config import get_settings from .db import Database from .embeddings import EmbeddingService -from .evals import HashHopEvaluator, PRDProofSuite from .ingestion import IngestionService from .memory import MemoryService from .profiles import runtime_profile_report +from .proofs import HashHopEvaluator, PRDProofSuite +from .proofs.contradictions import CORPUS_NAME, build_contradiction_scenarios, write_corpus +from .proofs.table import render_table from .research_modules import ResearchModuleService from .retrieval import RetrievalService from .runtime import QwenRuntime @@ -232,11 +234,19 @@ def proof_suite(cartridge_id: str | None = None) -> None: @app.command("latest-proof-suite") -def latest_proof_suite() -> None: +def latest_proof_suite(json_output: bool = typer.Option(False, "--json")) -> None: + """Print what the last proof suite claimed and what it actually proved.""" report = PRDProofSuite(get_settings()).latest() if report is None: + raise typer.BadParameter( + "no proof suite has been recorded yet; run: ely-eye proof-suite --cartridge-id " + ) + if json_output: + typer.echo(report.model_dump_json(indent=2)) + else: + typer.echo(render_table(report)) + if report.status == "failed": raise typer.Exit(code=1) - typer.echo(report.model_dump_json(indent=2)) @app.command("runtime-generation-proof") @@ -251,6 +261,14 @@ def visual_contradiction_proof() -> None: typer.echo(check.model_dump_json(indent=2)) +@app.command("contradiction-corpus") +def contradiction_corpus(data_dir: Path | None = None) -> None: + """Write the corpus the contradiction lens is graded on, so it can be read and replaced.""" + settings = get_settings() + target = write_corpus((data_dir or settings.data_dir) / CORPUS_NAME) + typer.echo(json_dump({"corpus": str(target), "scenarios": len(build_contradiction_scenarios())})) + + @app.command("finalize-cartridge") def finalize_cartridge(cartridge_id: str) -> None: report = CartridgeAssetService(get_settings()).finalize(cartridge_id) diff --git a/backend/src/ely_eye/compiler.py b/backend/src/ely_eye/compiler.py index 6447616..c58b58e 100644 --- a/backend/src/ely_eye/compiler.py +++ b/backend/src/ely_eye/compiler.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib from collections.abc import Iterable -from .cache_fabric import CacheFabric +from .cache_fabric import SEMANTIC_LAYER, CacheFabric from .config import Settings, get_settings from .db import Database from .retrieval import RetrievalService @@ -27,11 +27,18 @@ class ContextCompiler: def compile(self, question: str, profile: RuntimeProfile) -> CompiledContext: cache_key = self.cache.semantic_key(question, profile.value) cached = self.cache.read_semantic_cache(cache_key) - if cached: + if cached is not None: compiled = CompiledContext.model_validate(cached) if compiled.hits: + self.cache.record_hit(SEMANTIC_LAYER, cache_key) return compiled - + # A cached context with no evidence in it is not an answer, so counting it as a hit + # would let an empty cache entry look like a warm cache. + self.cache.record_miss( + SEMANTIC_LAYER, cache_key, metadata={"reason": "cached context has no hits"} + ) + else: + self.cache.record_miss(SEMANTIC_LAYER, cache_key) plan = self.plan(question, profile) hits = self.retrieval.search(plan.retrieval_query, plan.evidence_budget_atoms).hits packed_text, token_total = self._pack_hits(hits, plan.token_budget, plan.retrieval_query) diff --git a/backend/src/ely_eye/evals.py b/backend/src/ely_eye/evals.py deleted file mode 100644 index 297a8cf..0000000 --- a/backend/src/ely_eye/evals.py +++ /dev/null @@ -1,2427 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import math -import os -import random -import re -import secrets -import shutil -from pathlib import Path -from typing import Any - -import zstandard as zstd -from PIL import Image, ImageDraw - -from .benchmarks import ( - benchmark_prediction_report, - benchmark_registry_report, - benchmark_sample_report, - benchmark_source_report, -) -from .cartridge import CartridgeService -from .cartridge_assets import CartridgeAssetService, merkle_root -from .compiler import ContextCompiler -from .config import Settings, get_settings -from .db import Database -from .evaluation_schemas import ( - HashHopProof, - ProofCheck, - ProofSuiteReport, -) -from .evidence_graph import EvidenceGraphService -from .ingestion import SUPPORTED_CODE_EXTENSIONS, build_repository_context, find_git_root -from .profiles import runtime_profile_report -from .research_modules import ResearchModuleService -from .runtime import QwenRuntime, resolve_runtime_adapter -from .schemas import ( - AdapterManifest, - CartridgeManifest, - CompiledContext, - ContextPlan, - EvidenceAtom, - Modality, - RetrievalHit, - RuntimeProfile, - TrainingAdapterKind, - TrustScores, -) -from .training import ( - AUTOCAST_DTYPE, - GRADIENT_CHECKPOINTING_MODE, - TRAINING_FRAMEWORK, - TRAINING_METHOD, - TRAINING_PRECISION, - TRAINING_PROOF_VERSION, - safetensor_weight_stats, - sha256_file, - sha256_tree, -) -from .verifier import Verifier, VisualContradictionLens - - -class HashHopEvaluator: - def __init__(self, settings: Settings | None = None, runtime: QwenRuntime | None = None) -> None: - self.settings = settings or get_settings() - self.runtime = runtime or QwenRuntime(self.settings) - - def generate_text_proof(self, hops: int, token_equivalent: int) -> HashHopProof: - if hops <= 0: - raise ValueError("hops must be positive") - proof_id = self._proof_id("text", hops, token_equivalent) - root = self.settings.data_dir / "eval_proofs" / proof_id - root.mkdir(parents=True, exist_ok=True) - chain = [secrets.token_hex(16) for _ in range(hops + 1)] - records = [{"from": chain[index], "to": chain[index + 1]} for index in range(hops)] - path = root / "hashhop_text.jsonl" - path.write_text("\n".join(json.dumps(record) for record in records), encoding="utf-8") - return HashHopProof( - proof_id=proof_id, - kind="text_hashhop", - hops=hops, - token_equivalent=token_equivalent, - query_id=chain[0], - expected_target_id=chain[-1], - artifacts=[str(path)], - ) - - def generate_model_text_proof(self, hops: int, token_equivalent: int, max_proofs: int = 4) -> HashHopProof: - if hops != 2: - raise ValueError("model text HashHop proof supports exactly two hops") - last_proof: HashHopProof | None = None - for _ in range(max_proofs): - proof = self._generate_model_text_proof_once(hops, token_equivalent) - if proof.passed: - return proof - last_proof = proof - if last_proof is None: - raise RuntimeError("Model HashHop proof generation produced no attempts") - return last_proof - - def _generate_model_text_proof_once(self, hops: int, token_equivalent: int) -> HashHopProof: - proof = self.generate_text_proof(hops, token_equivalent) - root = Path(proof.artifacts[0]).parent - chain_path = root / "hashhop_text.jsonl" - try: - chain_text = chain_path.read_text(encoding="utf-8") - except OSError as error: - raise ValueError(f"{chain_path}: the HashHop chain is unreadable ({error})") from error - try: - records = [json.loads(line) for line in chain_text.splitlines() if line.strip()] - except json.JSONDecodeError as error: - raise ValueError(f"{chain_path}: the HashHop chain holds a malformed line ({error})") from error - decoys = [ - {"from": secrets.token_hex(16), "to": secrets.token_hex(16)} - for _ in range(max(4, hops * 2)) - ] - shuffled = records + decoys - random.Random(proof.proof_id).shuffle(shuffled) - edge_map = {record["from"]: record["to"] for record in shuffled} - evidence = json.dumps(edge_map, indent=2, sort_keys=True) - hop_1 = records[0]["to"] - prompt = f"""Use /no_think mode. Return compact JSON only. - -Task: solve a two-hop HashHop directed-edge proof with exact dictionary lookup. - -Example: -edge_map = {{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "cccccccccccccccccccccccccccccccc"}} -start_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -hop_1 = edge_map[start_id] -answer = edge_map[hop_1] -JSON = {{"hop_1":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","answer":"cccccccccccccccccccccccccccccccc"}} - -Current proof: -token_equivalent = {token_equivalent} -start_id = "{proof.query_id}" -hops = {hops} -edge_map = {evidence} - -Evaluate: -hop_1 = edge_map["{proof.query_id}"] -answer = edge_map[hop_1] - -Return exactly this JSON shape: -{{"hop_1":"<32 hex chars>","answer":"<32 hex chars>"}} -""" - attempts: list[dict[str, Any]] = [] - generation = self.runtime.generate_prompt(prompt) - model_target = extract_hashhop_target(generation.text) - attempts.append( - { - "step": "two_hop_expression", - "raw_generation": generation.text, - "parsed_target": model_target, - "expected_target": proof.expected_target_id, - "passed": model_target == proof.expected_target_id, - "input_tokens": generation.input_tokens, - "output_tokens": generation.output_tokens, - } - ) - if model_target != proof.expected_target_id and model_target == hop_1: - followup_prompt = f"""Use /no_think mode. Return compact JSON only. - -Task: continue a HashHop proof from a model-derived first hop. - -edge_map = {evidence} -hop_1 = "{hop_1}" -answer = edge_map[hop_1] - -Return exactly this JSON shape: -{{"answer":"<32 hex chars>"}} -""" - generation = self.runtime.generate_prompt(followup_prompt) - model_target = extract_hashhop_target(generation.text) - attempts.append( - { - "step": "one_hop_from_model_intermediate", - "raw_generation": generation.text, - "parsed_target": model_target, - "expected_target": proof.expected_target_id, - "passed": model_target == proof.expected_target_id, - "input_tokens": generation.input_tokens, - "output_tokens": generation.output_tokens, - } - ) - proof.model_target_id = model_target - proof.passed = model_target == proof.expected_target_id - artifact = root / "model_text_hashhop.json" - proof.artifacts.append(str(artifact)) - write_json( - artifact, - { - "proof": proof.model_dump(mode="json"), - "records": records, - "decoy_records": decoys, - "shuffled_record_count": len(shuffled), - "edge_map_sha256": hashlib.sha256(evidence.encode("utf-8")).hexdigest(), - "model_attempts": attempts, - "raw_generation": generation.text, - "parsed_target": model_target, - "expected_target": proof.expected_target_id, - "passed": proof.passed, - "runtime": { - "backend": generation.backend, - "adapter_path": generation.adapter_path, - "adapter_kind": generation.adapter_kind, - "adapter_id": generation.adapter_id, - "adapter_sha256": generation.adapter_sha256, - "input_tokens": generation.input_tokens, - "output_tokens": generation.output_tokens, - }, - }, - ) - (root / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") - return proof - - def generate_model_visual_proof(self, hops: int, token_equivalent: int, max_proofs: int = 3) -> HashHopProof: - if hops not in {1, 2}: - raise ValueError("model visual HashHop proof supports one or two hops") - last_proof: HashHopProof | None = None - for _ in range(max_proofs): - if hops == 1: - proof = self._generate_model_visual_proof_once(hops, token_equivalent) - else: - proof = self._generate_model_visual_two_hop_proof_once(token_equivalent) - if proof.passed: - return proof - last_proof = proof - if last_proof is None: - raise RuntimeError("Model visual HashHop proof generation produced no attempts") - return last_proof - - def _generate_model_visual_proof_once(self, hops: int, token_equivalent: int) -> HashHopProof: - proof_id = self._proof_id("visual-model", hops, token_equivalent) - root = self.settings.data_dir / "eval_proofs" / proof_id - image_dir = root / "visual_blocks" - image_dir.mkdir(parents=True, exist_ok=True) - source_id = secrets.token_hex(16) - target_id = secrets.token_hex(16) - decoy_ids = [secrets.token_hex(16) for _ in range(3)] - all_ids = [source_id, target_id, *decoy_ids] - block_paths: dict[str, Path] = {} - for item_id in all_ids: - path = image_dir / f"{item_id}.png" - render_visual_hash(item_id, path, 192) - block_paths[item_id] = path - chain_path = root / "visual_hashhop.jsonl" - chain_path.write_text(json.dumps({"from": source_id, "to": target_id}), encoding="utf-8") - slots = ("top-left", "top-right", "bottom-left", "bottom-right") - candidates = [target_id, *decoy_ids] - random.Random(proof_id).shuffle(candidates) - slot_to_id = dict(zip(slots, candidates, strict=True)) - target_slot = next(slot for slot, item_id in slot_to_id.items() if item_id == target_id) - arena_path = root / "visual_hashhop_arena.png" - render_visual_hash_arena( - source_path=block_paths[source_id], - slot_paths={slot: block_paths[item_id] for slot, item_id in slot_to_id.items()}, - target_slot=target_slot, - output_path=arena_path, - ) - proof = HashHopProof( - proof_id=proof_id, - kind="visual_hashhop", - hops=hops, - token_equivalent=token_equivalent, - query_id=source_id, - expected_target_id=target_id, - artifacts=[*(str(path) for path in block_paths.values()), str(chain_path), str(arena_path)], - ) - prompt = """Use /no_think mode. Return compact JSON only. - -The attached image is a Visual HashHop arena. -The large visual hash block on the left is the source. -Four candidate visual hash blocks are arranged on the right. -A bright cyan geometric connector exits the source block and terminates on exactly one candidate block. -Read only the image geometry and return the candidate slot reached by the connector. -Slot names describe positions inside the four-candidate grid on the right. - -Allowed target_slot values: -top-left, top-right, bottom-left, bottom-right - -Return exactly this JSON shape: -{"target_slot":""} -""" - generation = self.runtime.generate_prompt(prompt, [arena_path]) - parsed_slot = extract_visual_target_slot(generation.text) - model_target = slot_to_id.get(parsed_slot) if parsed_slot else None - proof.model_target_id = model_target - proof.passed = model_target == proof.expected_target_id - artifact = root / "model_visual_hashhop.json" - proof.artifacts.append(str(artifact)) - write_json( - artifact, - { - "proof": proof.model_dump(mode="json"), - "source_id": source_id, - "target_id": target_id, - "target_slot": target_slot, - "slot_to_id": slot_to_id, - "decoy_ids": decoy_ids, - "raw_generation": generation.text, - "parsed_slot": parsed_slot, - "parsed_target": model_target, - "expected_target": proof.expected_target_id, - "passed": proof.passed, - "runtime": { - "backend": generation.backend, - "adapter_path": generation.adapter_path, - "adapter_kind": generation.adapter_kind, - "adapter_id": generation.adapter_id, - "adapter_sha256": generation.adapter_sha256, - "input_tokens": generation.input_tokens, - "output_tokens": generation.output_tokens, - }, - }, - ) - (root / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") - return proof - - def _generate_model_visual_two_hop_proof_once(self, token_equivalent: int) -> HashHopProof: - hops = 2 - proof_id = self._proof_id("visual-model", hops, token_equivalent) - root = self.settings.data_dir / "eval_proofs" / proof_id - image_dir = root / "visual_blocks" - image_dir.mkdir(parents=True, exist_ok=True) - source_id = secrets.token_hex(16) - intermediate_id = secrets.token_hex(16) - target_id = secrets.token_hex(16) - intermediate_decoys = [secrets.token_hex(16) for _ in range(3)] - target_decoys = [secrets.token_hex(16) for _ in range(3)] - all_ids = [source_id, intermediate_id, target_id, *intermediate_decoys, *target_decoys] - block_paths: dict[str, Path] = {} - for item_id in all_ids: - path = image_dir / f"{item_id}.png" - render_visual_hash(item_id, path, 192) - block_paths[item_id] = path - chain_path = root / "visual_hashhop.jsonl" - chain_path.write_text( - "\n".join( - [ - json.dumps({"from": source_id, "to": intermediate_id}), - json.dumps({"from": intermediate_id, "to": target_id}), - ] - ), - encoding="utf-8", - ) - slots = ("top-left", "top-right", "bottom-left", "bottom-right") - intermediate_candidates = [intermediate_id, *intermediate_decoys] - target_candidates = [target_id, *target_decoys] - random.Random(f"{proof_id}:intermediate").shuffle(intermediate_candidates) - random.Random(f"{proof_id}:target").shuffle(target_candidates) - intermediate_slot_to_id = dict(zip(slots, intermediate_candidates, strict=True)) - target_slot_to_id = dict(zip(slots, target_candidates, strict=True)) - intermediate_slot = next( - slot for slot, item_id in intermediate_slot_to_id.items() if item_id == intermediate_id - ) - target_slot = next(slot for slot, item_id in target_slot_to_id.items() if item_id == target_id) - arena_path = root / "visual_hashhop_two_hop_arena.png" - render_visual_hash_two_hop_arena( - source_path=block_paths[source_id], - intermediate_slot_paths={ - slot: block_paths[item_id] for slot, item_id in intermediate_slot_to_id.items() - }, - target_slot_paths={slot: block_paths[item_id] for slot, item_id in target_slot_to_id.items()}, - intermediate_slot=intermediate_slot, - target_slot=target_slot, - output_path=arena_path, - ) - proof = HashHopProof( - proof_id=proof_id, - kind="visual_hashhop", - hops=hops, - token_equivalent=token_equivalent, - query_id=source_id, - expected_target_id=target_id, - artifacts=[*(str(path) for path in block_paths.values()), str(chain_path), str(arena_path)], - ) - prompt = """Use /no_think mode. Return compact JSON only. - -The attached image is a two-hop Visual HashHop arena. -The source visual hash block is on the left. -Four intermediate candidate visual hash blocks are in the center. -Four final candidate visual hash blocks are on the right. -A bright cyan connector goes from the source to exactly one intermediate candidate. -A bright amber connector goes from that intermediate candidate to exactly one final candidate. -Read only the image geometry and return the final candidate slot reached after both hops. -The intermediate_slot value describes the center grid. The target_slot value describes the rightmost final-candidate grid. - -Allowed slot values: -top-left, top-right, bottom-left, bottom-right - -Return exactly this JSON shape: -{"intermediate_slot":"","target_slot":""} -""" - generation = self.runtime.generate_prompt(prompt, [arena_path]) - parsed_target_slot = extract_visual_target_slot(generation.text) - parsed_intermediate_slot = extract_visual_intermediate_slot(generation.text) - model_target = target_slot_to_id.get(parsed_target_slot) if parsed_target_slot else None - proof.model_target_id = model_target - proof.passed = model_target == proof.expected_target_id - artifact = root / "model_visual_hashhop.json" - proof.artifacts.append(str(artifact)) - write_json( - artifact, - { - "proof": proof.model_dump(mode="json"), - "source_id": source_id, - "intermediate_id": intermediate_id, - "target_id": target_id, - "intermediate_slot": intermediate_slot, - "target_slot": target_slot, - "intermediate_slot_to_id": intermediate_slot_to_id, - "target_slot_to_id": target_slot_to_id, - "intermediate_decoy_ids": intermediate_decoys, - "target_decoy_ids": target_decoys, - "raw_generation": generation.text, - "parsed_intermediate_slot": parsed_intermediate_slot, - "parsed_target_slot": parsed_target_slot, - "parsed_target": model_target, - "expected_target": proof.expected_target_id, - "passed": proof.passed, - "runtime": { - "backend": generation.backend, - "adapter_path": generation.adapter_path, - "adapter_kind": generation.adapter_kind, - "adapter_id": generation.adapter_id, - "adapter_sha256": generation.adapter_sha256, - "input_tokens": generation.input_tokens, - "output_tokens": generation.output_tokens, - }, - }, - ) - (root / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") - return proof - - def generate_visual_proof(self, hops: int, token_equivalent: int, tile_size: int = 256) -> HashHopProof: - if hops <= 0: - raise ValueError("hops must be positive") - proof_id = self._proof_id("visual", hops, token_equivalent) - root = self.settings.data_dir / "eval_proofs" / proof_id - image_dir = root / "visual_blocks" - image_dir.mkdir(parents=True, exist_ok=True) - ids = [secrets.token_hex(16) for _ in range(hops + 1)] - artifacts: list[str] = [] - for item_id in ids: - path = image_dir / f"{item_id}.png" - render_visual_hash(item_id, path, tile_size) - artifacts.append(str(path)) - chain_path = root / "visual_hashhop.jsonl" - chain_path.write_text( - "\n".join(json.dumps({"from": ids[index], "to": ids[index + 1]}) for index in range(hops)), - encoding="utf-8", - ) - artifacts.append(str(chain_path)) - return HashHopProof( - proof_id=proof_id, - kind="visual_hashhop", - hops=hops, - token_equivalent=token_equivalent, - query_id=ids[0], - expected_target_id=ids[-1], - artifacts=artifacts, - ) - - def _proof_id(self, kind: str, hops: int, token_equivalent: int) -> str: - raw = f"{kind}:{hops}:{token_equivalent}:{secrets.token_hex(8)}" - return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] - - -class PRDProofSuite: - required_adapters = ( - TrainingAdapterKind.hme_core_lora, - TrainingAdapterKind.hme_vision_lora, - TrainingAdapterKind.hme_ttt_vl, - TrainingAdapterKind.hme_visual_mtp, - TrainingAdapterKind.hme_router, - TrainingAdapterKind.hme_retrieval, - ) - - def __init__(self, settings: Settings | None = None) -> None: - self.settings = settings or get_settings() - self.db = Database(self.settings) - self.cartridges = CartridgeService(self.settings, self.db) - - def run(self, cartridge_id: str | None = None) -> ProofSuiteReport: - row = self._resolve_cartridge(cartridge_id) - cartridge_id = str(row["cartridge_id"]) - root = Path(row["root_path"]) - manifest = CartridgeManifest.model_validate_json(row["manifest_json"]) - proof_id = self._proof_id(cartridge_id, manifest.dna) - proof_root = self.settings.data_dir / "eval_proofs" / proof_id - proof_root.mkdir(parents=True, exist_ok=True) - dna_before = manifest.dna - - asset_check, manifest = self._cartridge_assets_check(cartridge_id, root, proof_root) - checks = [ - self._adapter_training_check(cartridge_id, root, proof_root), - self._runtime_adapter_check(proof_root), - self._runtime_generation_check(proof_root), - asset_check, - self._memory_capsule_check(root, proof_root), - self._cartridge_replay_check(cartridge_id, root, manifest, proof_root), - self._visual_hashhop_check(proof_root), - self._standard_benchmark_check(proof_root), - self._repository_input_check(proof_root), - self._runtime_profile_check(proof_root), - self._research_module_check(proof_root), - self._learned_memory_recall_check(cartridge_id, root, proof_root), - self._memory_dna_check(root, manifest, proof_root), - self._cache_trace_check(proof_root), - self._visual_contradiction_check(proof_root), - ] - status = "passed" if all(check.status == "passed" for check in checks) else "failed" - report = ProofSuiteReport( - proof_id=proof_id, - cartridge_id=cartridge_id, - status=status, - checks=checks, - artifacts={}, - dna_before=dna_before, - dna_after=dna_before, - ) - report_path = proof_root / "proof_suite.json" - report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") - report.artifacts = relative_artifacts(proof_root) - report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") - attached_manifest = self.cartridges.attach_eval_proof(cartridge_id, proof_root) - report.dna_after = attached_manifest.dna - report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") - cartridge_report = root / "eval_proofs" / proof_id / "proof_suite.json" - cartridge_report.write_text(report.model_dump_json(indent=2), encoding="utf-8") - return report - - def runtime_generation_proof(self) -> ProofCheck: - proof_id = f"runtime_generation_{secrets.token_hex(8)}" - proof_root = self.settings.data_dir / "eval_proofs" / proof_id - proof_root.mkdir(parents=True, exist_ok=True) - return self._runtime_generation_check(proof_root) - - def visual_contradiction_proof(self) -> ProofCheck: - proof_id = f"visual_contradiction_{secrets.token_hex(8)}" - proof_root = self.settings.data_dir / "eval_proofs" / proof_id - proof_root.mkdir(parents=True, exist_ok=True) - return self._visual_contradiction_check(proof_root) - - def latest(self) -> ProofSuiteReport | None: - proof_root = self.settings.data_dir / "eval_proofs" - reports = sorted( - proof_root.glob("prd_proof_suite_*/proof_suite.json"), - key=lambda path: path.stat().st_mtime, - reverse=True, - ) - if not reports: - return None - return ProofSuiteReport.model_validate_json(reports[0].read_text(encoding="utf-8")) - - def _adapter_training_check(self, cartridge_id: str, cartridge_root: Path, proof_root: Path) -> ProofCheck: - rows: list[dict[str, Any]] = [] - missing: list[str] = [] - manifests = self._adapter_manifests(cartridge_id) - for kind in self.required_adapters: - item = manifests.get(kind) - if item is None: - missing.append(kind.value) - continue - manifest, adapter_root = item - weight_path = adapter_root / "adapter_model.safetensors" - dataset_path = Path(manifest.dataset_path) - cartridge_adapter = cartridge_root / "adapters" / kind.value / "adapter_manifest.json" - weight_report = safetensor_weight_stats(weight_path) - keys = int(weight_report["tensor_count"]) - summary = read_json_or_empty(adapter_root / "training_summary.json") - summary_weight_sha = summary.get("adapter_weight_sha256") - summary_weight_sha_ok = ( - isinstance(summary_weight_sha, str) and summary_weight_sha == weight_report["sha256"] - ) - manifest_weight_sha_ok = ( - isinstance(manifest.adapter_weight_sha256, str) - and manifest.adapter_weight_sha256 == weight_report["sha256"] - ) - manifest_weight_stats_ok = ( - manifest.adapter_tensor_count == weight_report["tensor_count"] - and manifest.adapter_lora_tensor_count == weight_report["lora_tensor_count"] - and manifest.adapter_total_elements == weight_report["total_elements"] - and manifest.adapter_nonzero_elements == weight_report["nonzero_elements"] - and manifest.adapter_nonzero_elements > 0 - and manifest.adapter_lora_tensor_count > 0 - and manifest.adapter_weights_finite - ) - peft_ok = peft_config_loads(adapter_root) - dataset_sha_ok = dataset_path.exists() and sha256_file(dataset_path) == manifest.dataset_sha256 - tree_sha_ok = sha256_tree(adapter_root) == manifest.sha256 - cartridge_bound = cartridge_adapter.exists() - trace_report = validate_training_trace(manifest, adapter_root) - proof_report = validate_training_proof(manifest, adapter_root) - contract_report = validate_training_contract(manifest) - valid = ( - manifest.sample_count > 0 - and manifest.max_steps > 0 - and manifest.trainable_params > 0 - and manifest.final_loss is not None - and weight_report["valid"] - and summary_weight_sha_ok - and manifest_weight_sha_ok - and manifest_weight_stats_ok - and peft_ok - and dataset_sha_ok - and tree_sha_ok - and cartridge_bound - and trace_report["valid"] - and proof_report["valid"] - and contract_report["valid"] - ) - if not valid: - missing.append(kind.value) - rows.append( - { - "kind": kind.value, - "adapter": str(adapter_root), - "dataset": str(dataset_path), - "sample_count": manifest.sample_count, - "max_steps": manifest.max_steps, - "trainable_params": manifest.trainable_params, - "final_loss": manifest.final_loss, - "safetensor_keys": keys, - "adapter_weight_sha256": weight_report["sha256"], - "adapter_weight_sha256_ok": summary_weight_sha_ok and manifest_weight_sha_ok, - "adapter_weight_stats_ok": manifest_weight_stats_ok, - "adapter_weight_stats": weight_report, - "dataset_sha256_ok": dataset_sha_ok, - "adapter_sha256_ok": tree_sha_ok, - "peft_config_ok": peft_ok, - "cartridge_bound": cartridge_bound, - "training_contract": contract_report, - "training_trace": trace_report, - "training_proof": proof_report, - } - ) - artifact = proof_root / "adapter_training.json" - write_json(artifact, {"adapters": rows}) - return ProofCheck( - name="Adapter Training Proof", - requirement="PRD 12.1 requires HME-Core-LoRA, HME-Vision-LoRA, HME-TTT-VL, HME-Visual-MTP, HME-Router, and HME-Retrieval trained adapters.", - status="failed" if missing else "passed", - evidence=[str(artifact), str(self.settings.adapters_dir), str(cartridge_root / "adapters")], - detail="verified CUDA BF16 optimizer updates and nonzero LoRA safetensors: " - + ", ".join(row["kind"] for row in rows), - ) - - def _runtime_adapter_check(self, proof_root: Path) -> ProofCheck: - runtime = QwenRuntime(self.settings) - status = runtime.status() - adapter = resolve_runtime_adapter(self.settings) - weight_path = adapter.path / "adapter_model.safetensors" if adapter else None - payload = { - "runtime_status": status.model_dump(mode="json"), - "resolved_adapter": { - "path": str(adapter.path) if adapter else None, - "kind": adapter.manifest.kind.value if adapter else None, - "adapter_id": adapter.manifest.adapter_id if adapter else None, - "sha256": adapter.manifest.sha256 if adapter else None, - "base_model": adapter.manifest.base_model if adapter else None, - "weight_path": str(weight_path) if weight_path else None, - "safetensor_keys": count_safetensor_keys(weight_path) if weight_path else 0, - "peft_config_ok": peft_config_loads(adapter.path) if adapter else False, - }, - } - artifact = proof_root / "runtime_adapter.json" - write_json(artifact, payload) - valid = ( - status.available - and adapter is not None - and adapter.manifest.kind == TrainingAdapterKind.hme_core_lora - and adapter.manifest.base_model == self.settings.model_id - and status.adapter_path == str(adapter.path) - and weight_path is not None - and count_safetensor_keys(weight_path) > 0 - and peft_config_loads(adapter.path) - ) - return ProofCheck( - name="Runtime Adapter Binding", - requirement="PRD 18 requires the local model package to run with the trained adapter, and PRD 12.1 requires HME-Core-LoRA to be usable by the Qwen runtime.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail=( - f"{status.backend} runtime resolves {adapter.manifest.kind.value} at {adapter.path}" - if adapter - else "runtime adapter resolution is empty" - ), - ) - - def _runtime_generation_check(self, proof_root: Path) -> ProofCheck: - artifact = proof_root / "runtime_generation.json" - retrieval_question = ( - "Using the Ely-Eye PRD evidence, state the PRD 12.1 local training adapter " - "requirements for HME-Core-LoRA, HME-Vision-LoRA, HME-TTT-VL, " - "HME-Visual-MTP, HME-Router, and HME-Retrieval. Return JSON with atom id " - "citations." - ) - proof_settings = self.settings.model_copy(update={"max_new_tokens": 256, "temperature": 0.0}) - runtime = QwenRuntime(proof_settings) - payload: dict[str, Any] = { - "question": retrieval_question, - "status": "failed", - } - try: - compiler = ContextCompiler(proof_settings, self.db) - context = compiler.compile(retrieval_question, RuntimeProfile.library_100m) - context = compact_generation_context(context, "12.1") - if not context.hits: - raise RuntimeError("No PRD text evidence was available for runtime generation proof.") - primary_atom_id = context.hits[0].atom.atom_id - generation_question = ( - 'Return one compact JSON object. Put the "citations" field first and set it to ' - f'["{primary_atom_id}"]. In one short sentence, state that PRD 12.1 trains ' - "HME-Core-LoRA, HME-Vision-LoRA, HME-TTT-VL, HME-Visual-MTP, HME-Router, " - "and HME-Retrieval." - ) - payload["generation_question"] = generation_question - status = runtime.status() - adapter = resolve_runtime_adapter(proof_settings) - generation = runtime.generate(generation_question, context) - answer, citations, citation_report = Verifier().verify_answer(generation.text, context) - claim_id = EvidenceGraphService(proof_settings, self.db).record_answer( - generation_question, - answer, - citation_report, - context, - ) - payload.update( - { - "status": "passed", - "runtime_status": status.model_dump(mode="json"), - "adapter": { - "path": generation.adapter_path, - "kind": generation.adapter_kind, - "adapter_id": generation.adapter_id, - "sha256": generation.adapter_sha256, - }, - "tokens": { - "input": generation.input_tokens, - "output": generation.output_tokens, - }, - "context": { - "profile": context.plan.profile.value, - "hit_ids": [hit.atom.atom_id for hit in context.hits], - "token_equivalent": context.token_equivalent, - "cache_trace_id": context.cache_trace_id, - }, - "raw_generation": generation.text, - "verified_answer": answer, - "citations": citations, - "evidence_graph_claim_id": claim_id, - "verifier": citation_report.model_dump(mode="json"), - } - ) - valid = ( - status.available - and generation.backend == "transformers" - and adapter is not None - and generation.adapter_path == str(adapter.path) - and generation.adapter_kind == TrainingAdapterKind.hme_core_lora.value - and generation.adapter_sha256 == adapter.manifest.sha256 - and bool(context.hits) - and bool(generation.text.strip()) - and (generation.output_tokens or 0) > 0 - and citation_report.citation_accuracy > 0.0 - ) - if not valid: - payload["status"] = "failed" - write_json(artifact, payload) - return ProofCheck( - name="Runtime Adapter Generation", - requirement="PRD 12.1 and PRD 18 require the trained HME-Core-LoRA adapter to be loaded by the local Qwen runtime and used for an evidence-cited generation.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail=( - f"{generation.backend} generated {generation.output_tokens} tokens with " - f"{generation.adapter_kind} citation accuracy " - f"{citation_report.citation_accuracy:.2f}" - ), - ) - except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run - payload["error"] = str(exc) - write_json(artifact, payload) - return ProofCheck( - name="Runtime Adapter Generation", - requirement="PRD 12.1 and PRD 18 require the trained HME-Core-LoRA adapter to be loaded by the local Qwen runtime and used for an evidence-cited generation.", - status="failed", - evidence=[str(artifact)], - detail=f"runtime generation failed: {exc}", - ) - finally: - runtime.unload() - - def _cartridge_assets_check( - self, - cartridge_id: str, - cartridge_root: Path, - proof_root: Path, - ) -> tuple[ProofCheck, CartridgeManifest]: - report = CartridgeAssetService(self.settings, self.db).finalize(cartridge_id) - source_report = cartridge_root / "cartridge_assets.json" - proof_report = proof_root / "cartridge_assets.json" - shutil.copy2(source_report, proof_report) - rows = self.db.list_cartridges() - manifest_json = next(row["manifest_json"] for row in rows if row["cartridge_id"] == cartridge_id) - manifest = CartridgeManifest.model_validate_json(manifest_json) - expected = { - "text_vectors", - "visual_vectors", - "kv_snapshots", - "memory_capsule", - "memory_capsule_index", - "ttt_vl_adapter", - "visual_mtp_head", - "cartridge_assets", - } - present = expected <= set(manifest.artifacts) - by_name = {asset.name: asset for asset in report.assets} - visual_vectors = by_name.get("visual_vectors") - memory_capsule = by_name.get("memory_capsule") - valid = ( - report.status == "ready" - and present - and all(asset.size_bytes > 0 for asset in report.assets) - and visual_vectors is not None - and visual_vectors.item_count > 0 - and memory_capsule is not None - and memory_capsule.item_count > 0 - ) - return ( - ProofCheck( - name="Cartridge Physical Assets", - requirement="PRD 8.3 requires text vectors, non-empty visual vectors, KV snapshots, a 100M memory capsule, TTT-VL adapter weights, Visual-MTP head weights, and eval proofs inside the Context Cartridge.", - status="passed" if valid else "failed", - evidence=[str(proof_report), str(cartridge_root)], - detail="materialized assets: " + ", ".join(asset.name for asset in report.assets), - ), - manifest, - ) - - def _memory_capsule_check(self, cartridge_root: Path, proof_root: Path) -> ProofCheck: - capsule_path = cartridge_root / "memory_capsule.json.zst" - index_path = cartridge_root / "memory_capsule_index.json" - artifact = proof_root / "memory_capsule.json" - payload: dict[str, Any] = { - "capsule_path": str(capsule_path), - "index_path": str(index_path), - "status": "failed", - } - valid = False - try: - capsule = read_memory_capsule(capsule_path) - index = json.loads(index_path.read_text(encoding="utf-8")) if index_path.exists() else {} - segments = capsule.get("segments") if isinstance(capsule.get("segments"), list) else [] - segment_hashes = [validated_segment_hash(segment) for segment in segments] - token_total = sum(int(segment.get("token_count") or 0) for segment in segments) - computed_merkle = merkle_root(segment_hashes) - valid = ( - capsule.get("format") == "ely-eye-memory-capsule-v1" - and int(capsule.get("token_equivalent") or 0) >= self.settings.library_target_tokens - and int(capsule.get("segment_count") or 0) == len(segments) - and token_total == int(capsule.get("token_equivalent") or -1) - and computed_merkle == capsule.get("segment_merkle_root") - and index.get("segment_merkle_root") == capsule.get("segment_merkle_root") - and int(index.get("token_equivalent") or 0) == int(capsule.get("token_equivalent") or -1) - and int(capsule.get("source_atom_count") or 0) > 0 - and all(segment_hashes) - ) - payload.update( - { - "status": "passed" if valid else "failed", - "token_equivalent": capsule.get("token_equivalent"), - "segment_count": capsule.get("segment_count"), - "source_atom_count": capsule.get("source_atom_count"), - "source_token_equivalent": capsule.get("source_token_equivalent"), - "segment_merkle_root": capsule.get("segment_merkle_root"), - "computed_merkle_root": computed_merkle, - "first_segment": segments[0] if segments else None, - "last_segment": segments[-1] if segments else None, - } - ) - except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run - payload["error"] = str(exc) - write_json(artifact, payload) - return ProofCheck( - name="100M Memory Capsule", - requirement="PRD 2.2, 7.3, 8.3, and 14.3 require a 10M-100M token-equivalent Context Cartridge memory capsule with verifiable retrieval commitments.", - status="passed" if valid else "failed", - evidence=[str(artifact), str(capsule_path), str(index_path)], - detail=( - f"{payload.get('token_equivalent')} token-equivalent memory capsule with " - f"{payload.get('segment_count')} committed segments" - ), - ) - - def _cartridge_replay_check( - self, - cartridge_id: str, - cartridge_root: Path, - manifest: CartridgeManifest, - proof_root: Path, - ) -> ProofCheck: - atoms_path = cartridge_root / manifest.artifacts["atoms"] - sources_path = cartridge_root / manifest.artifacts["sources"] - sparse_path = cartridge_root / manifest.artifacts["sparse_index"] - graph_path = cartridge_root / manifest.artifacts["temporal_graph"] - computed_dna = self.cartridges.compute_dna(cartridge_root) - atom_rows = parquet_row_count(atoms_path) - try: - sources = json.loads(sources_path.read_text(encoding="utf-8")) if sources_path.exists() else [] - except (OSError, json.JSONDecodeError) as error: - raise ValueError(f"{sources_path}: the cartridge source list is unreadable ({error})") from error - question = "Which Ely-Eye PRD requirements prove long context memory, HashHop, and adapters?" - compiler = ContextCompiler(self.settings, self.db) - replay_a = compiler.compile(question, RuntimeProfile.library_100m) - replay_b = compiler.compile(question, RuntimeProfile.library_100m) - digest_a = compiled_context_digest(replay_a) - digest_b = compiled_context_digest(replay_b) - payload = { - "cartridge_id": cartridge_id, - "manifest_atom_count": manifest.atom_count, - "parquet_atom_count": atom_rows, - "manifest_source_count": manifest.source_count, - "source_count": len(sources), - "dna_manifest": manifest.dna, - "dna_computed": computed_dna, - "artifacts": { - "atoms": str(atoms_path), - "sources": str(sources_path), - "sparse_index": str(sparse_path), - "temporal_graph": str(graph_path), - }, - "replay": { - "question": question, - "digest_a": digest_a, - "digest_b": digest_b, - "hit_ids_a": [hit.atom.atom_id for hit in replay_a.hits], - "hit_ids_b": [hit.atom.atom_id for hit in replay_b.hits], - "cache_trace_a": replay_a.cache_trace_id, - "cache_trace_b": replay_b.cache_trace_id, - }, - } - artifact = proof_root / "cartridge_replay.json" - write_json(artifact, payload) - valid = ( - atoms_path.exists() - and sources_path.exists() - and sparse_path.exists() - and graph_path.exists() - and atom_rows == manifest.atom_count - and len(sources) == manifest.source_count - and computed_dna == manifest.dna - and bool(replay_a.hits) - and digest_a == digest_b - ) - return ProofCheck( - name="Context Cartridge Replay", - requirement="PRD 14.2 requires the same cartridge to load reproducibly and produce consistent replay evidence.", - status="passed" if valid else "failed", - evidence=[str(artifact), str(cartridge_root / "manifest.json")], - detail=f"{atom_rows} atoms, {len(sources)} sources, replay digest {digest_a}", - ) - - def _visual_hashhop_check(self, proof_root: Path) -> ProofCheck: - proof_settings = self.settings.model_copy(update={"max_new_tokens": 128, "temperature": 0.0}) - runtime = QwenRuntime(proof_settings) - evaluator = HashHopEvaluator(proof_settings, runtime) - try: - base_proofs = [ - evaluator.generate_visual_proof(1, 262_144), - evaluator.generate_visual_proof(2, 262_144), - evaluator.generate_visual_proof(1, 1_010_000), - evaluator.generate_visual_proof(1, 100_000_000), - evaluator.generate_text_proof(2, 1_010_000), - evaluator.generate_model_text_proof(2, 1_010_000), - ] - model_visual_1hop = collect_model_visual_proofs(evaluator, 1, 262_144, 3, 0.90, 4) - model_visual_2hop = collect_model_visual_proofs(evaluator, 2, 262_144, 3, 0.75, 6) - proofs = base_proofs + model_visual_1hop + model_visual_2hop - finally: - runtime.unload() - payloads: list[dict[str, Any]] = [] - for proof in proofs: - target = proof_root / "hashhop" / proof.proof_id - copy_proof_artifacts(proof, target) - payloads.append( - { - "proof": proof.model_dump(mode="json"), - "chain_valid": hashhop_chain_valid(proof), - "artifact_dir": str(target), - } - ) - artifact = proof_root / "visual_hashhop_suite.json" - model_solved = [ - item - for item in payloads - if item["proof"]["model_target_id"] is not None or item["proof"]["passed"] is not None - ] - model_text_solved = [ - proof - for proof in base_proofs - if proof.kind == "text_hashhop" and proof.model_target_id is not None - ] - model_solved_passed = [item for item in model_solved if item["proof"]["passed"]] - visual_1hop_rate = pass_rate(model_visual_1hop) - visual_2hop_rate = pass_rate(model_visual_2hop) - metrics = { - "visual_hashhop_262k_1hop": { - "target": 0.90, - "sample_count": len(model_visual_1hop), - "passed": sum(1 for proof in model_visual_1hop if proof.passed), - "pass_rate": visual_1hop_rate, - }, - "visual_hashhop_262k_2hop": { - "target": 0.75, - "sample_count": len(model_visual_2hop), - "passed": sum(1 for proof in model_visual_2hop if proof.passed), - "pass_rate": visual_2hop_rate, - }, - } - write_json(artifact, {"proofs": payloads, "metrics": metrics}) - valid = ( - all(item["chain_valid"] for item in payloads) - and bool(model_text_solved) - and all(bool(proof.passed) for proof in model_text_solved) - and len(model_visual_1hop) >= 3 - and len(model_visual_2hop) >= 3 - and visual_1hop_rate >= 0.90 - and visual_2hop_rate >= 0.75 - ) - return ProofCheck( - name="Visual HashHop", - requirement="PRD 14.2 and 14.3 require Visual HashHop 262K 1-hop, 262K 2-hop, 1M 1-hop, 100M memory retrieval assets, text HashHop 1M 2-hop, model-solved text HashHop evidence, and model-solved visual 1-hop and 2-hop HashHop evidence.", - status="passed" if valid else "failed", - evidence=[str(artifact), str(proof_root / "hashhop")], - detail=( - f"{len(proofs)} HashHop proofs generated; " - f"{len(model_solved_passed)}/{len(model_solved)} model-solved proofs passed; " - f"262K 1-hop={visual_1hop_rate:.2f}, 2-hop={visual_2hop_rate:.2f}" - ), - ) - - def _standard_benchmark_check(self, proof_root: Path) -> ProofCheck: - registry = benchmark_registry_report() - sources = benchmark_source_report() - samples = benchmark_sample_report(proof_root / "benchmark_samples") - predictions = benchmark_prediction_report(self.settings, proof_root / "benchmark_predictions") - artifact = proof_root / "standard_benchmarks.json" - source_artifact = proof_root / "standard_benchmark_sources.json" - sample_artifact = proof_root / "standard_benchmark_samples.json" - prediction_artifact = proof_root / "standard_benchmark_predictions.json" - artifact.write_text(registry.model_dump_json(indent=2), encoding="utf-8") - source_artifact.write_text(sources.model_dump_json(indent=2), encoding="utf-8") - sample_artifact.write_text(samples.model_dump_json(indent=2), encoding="utf-8") - prediction_artifact.write_text(predictions.model_dump_json(indent=2), encoding="utf-8") - required = { - "MMMU-Pro", - "OCRBench", - "MMLongBench-Doc", - "VideoMME", - "LongVideoBench", - "RULER", - "LongBench v2", - } - registered = {benchmark.name for benchmark in registry.benchmarks} - source_names = {source.name for source in sources.sources if source.status == "ready" and source.sha} - loaded_sample_names = {sample.name for sample in samples.samples if sample.status == "loaded"} - gated_sample_names = {sample.name for sample in samples.samples if sample.status == "gated"} - source_proved_sample_names = { - sample.name - for sample in samples.samples - if sample.status == "loaded" and sample.sample_sha256 and sample.source_dataset and sample.source_sha - } - predicted_sample_names = { - prediction.name for prediction in predictions.predictions if prediction.status == "predicted" - } - prediction_scores = { - prediction.name: prediction.score - for prediction in predictions.predictions - if prediction.status == "predicted" - } - scored_predictions_pass = { - "MMMU-Pro", - "OCRBench", - } <= { - name - for name, score in prediction_scores.items() - if isinstance(score, (int, float)) and math.isfinite(float(score)) and float(score) >= 0.5 - } - sample_names = loaded_sample_names | gated_sample_names - valid = ( - registry.status == "ready" - and sources.status == "ready" - and samples.status == "ready" - and predictions.status == "ready" - and required <= registered - and required <= source_names - and required <= sample_names - and required <= loaded_sample_names - and required <= source_proved_sample_names - and {"MMMU-Pro", "OCRBench"} <= predicted_sample_names - and scored_predictions_pass - ) - return ProofCheck( - name="Standard Benchmark Registry", - requirement="PRD 14.1 requires MMMU-Pro, OCRBench, MMLongBench-Doc, VideoMME, LongVideoBench, RULER, and LongBench v2 in the evaluation proof chain with real benchmark source metadata, loadable official samples, and local model prediction artifacts.", - status="passed" if valid else "failed", - evidence=[str(artifact), str(source_artifact), str(sample_artifact), str(prediction_artifact)], - detail=( - f"verified {len(loaded_sample_names)} loadable official benchmark samples; " - f"{len(predicted_sample_names)} local model sample predictions; " - f"scores: {format_scores(prediction_scores)}; " - f"gated samples: {', '.join(sorted(gated_sample_names)) or 'none'}" - ), - ) - - def _repository_input_check(self, proof_root: Path) -> ProofCheck: - root = project_root() - files = discover_repository_files(root) - code_files = [path for path in files if path.suffix.lower() in SUPPORTED_CODE_EXTENSIONS] - context = build_repository_context(root, files, code_files, find_git_root(root)) - artifact = proof_root / "repository_input.json" - write_json(artifact, context) - call_graph = context["code"]["call_graph"] - git_context = context["git"] - issue_pr_history = context["issue_pr_history"] - valid = ( - context["structure"]["indexed_file_count"] > 0 - and context["code"]["indexed_file_count"] > 0 - and bool(call_graph) - and bool(git_context.get("available")) - and bool(git_context.get("recent_commits")) - and issue_pr_history.get("source") == "local_git_commit_history" - ) - return ProofCheck( - name="Repository Input Coverage", - requirement="PRD 10.1 requires code repository structure indexing, call graph extraction, and issue/PR history association.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail=( - f"{context['structure']['indexed_file_count']} files indexed; " - f"{context['code']['indexed_file_count']} code files; " - f"{len(call_graph)} call edges; " - f"{len(git_context.get('recent_commits') or [])} git commits" - ), - ) - - def _runtime_profile_check(self, proof_root: Path) -> ProofCheck: - report = runtime_profile_report() - artifact = proof_root / "runtime_profiles.json" - artifact.write_text(report.model_dump_json(indent=2), encoding="utf-8") - required_runtimes = {"SGLang", "vLLM", "KTransformers", "llama.cpp"} - runtimes = {profile.runtime for profile in report.profiles} - valid = report.status == "ready" and required_runtimes <= runtimes - return ProofCheck( - name="Runtime Profile Delivery", - requirement="PRD 13.1 requires SGLang as the main runtime plus vLLM, KTransformers, and llama.cpp deployment profiles.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail="registered runtimes: " + ", ".join(profile.name for profile in report.profiles), - ) - - def _research_module_check(self, proof_root: Path) -> ProofCheck: - query = "Ely-Eye V-NSA DVKV x-modal KV speculative prefetch evidence graph" - compiler = ContextCompiler(self.settings, self.db) - context = compiler.compile(query, RuntimeProfile.research_theater) - artifact = proof_root / "research_modules.json" - report = ResearchModuleService(self.settings, self.db).build_report(query, context.hits) - module_checks = research_module_checks(report) - write_json(artifact, {"modules": report, "checks": module_checks}) - valid = all(check["passed"] for check in module_checks) - return ProofCheck( - name="Research Module Delivery", - requirement="PRD 11 requires V-NSA, DVKV, X-Modal KV Sharing, Speculative Context Prefetch, Predictive KV Eviction, Learned Evidence Graph, Differentiable Retrieval Adapter, and Sleep-Time Visual Consolidation.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail="verified executable research modules: " + ", ".join(check["module"] for check in module_checks), - ) - - def _learned_memory_recall_check( - self, - cartridge_id: str, - cartridge_root: Path, - proof_root: Path, - ) -> ProofCheck: - manifests = self._adapter_manifests(cartridge_id) - learned_kinds = ( - TrainingAdapterKind.hme_ttt_vl, - TrainingAdapterKind.hme_core_lora, - TrainingAdapterKind.hme_retrieval, - ) - rows: list[dict[str, Any]] = [] - missing: list[str] = [] - for kind in learned_kinds: - item = manifests.get(kind) - if item is None: - missing.append(kind.value) - continue - manifest, adapter_root = item - weight_path = adapter_root / "adapter_model.safetensors" - cartridge_weight = cartridge_root / "adapters" / kind.value / "adapter_model.safetensors" - rows.append( - { - "kind": kind.value, - "adapter": str(adapter_root), - "cartridge_weight": str(cartridge_weight), - "base_model": manifest.base_model, - "dataset_sha256": manifest.dataset_sha256, - "sample_count": manifest.sample_count, - "trainable_params": manifest.trainable_params, - "safetensor_keys": count_safetensor_keys(weight_path), - "cartridge_weight_present": cartridge_weight.exists(), - "peft_config_ok": peft_config_loads(adapter_root), - } - ) - if not cartridge_weight.exists() or count_safetensor_keys(weight_path) <= 0: - missing.append(kind.value) - artifact = proof_root / "learned_memory_recall.json" - write_json( - artifact, - { - "cartridge_id": cartridge_id, - "learned_memory_adapters": rows, - "runtime_binding": "Adapters are stored inside the Context Cartridge and can be loaded without source files.", - }, - ) - return ProofCheck( - name="Learned Memory Recall", - requirement="PRD 14.2 and 15.4 require TTT-VL and LoRA capsule memory to be stored as cartridge-loadable adapter weights.", - status="failed" if missing else "passed", - evidence=[str(artifact), str(cartridge_root / "adapters" / TrainingAdapterKind.hme_ttt_vl.value)], - detail="verified cartridge-loadable learned memory adapters: " + ", ".join(row["kind"] for row in rows), - ) - - def _memory_dna_check( - self, - cartridge_root: Path, - manifest: CartridgeManifest, - proof_root: Path, - ) -> ProofCheck: - computed = self.cartridges.compute_dna(cartridge_root) - payload = { - "manifest_dna": manifest.dna, - "computed_dna": computed, - "formula": "sha256(sorted cartridge artifacts excluding manifest.json)", - "artifact_count": sum(1 for path in cartridge_root.rglob("*") if path.is_file()), - } - artifact = proof_root / "memory_dna.json" - write_json(artifact, payload) - return ProofCheck( - name="Memory DNA", - requirement="PRD 11.14 requires a reproducible Memory DNA over adapter delta, evidence graph, retrieval index, and eval proofs.", - status="passed" if manifest.dna == computed else "failed", - evidence=[str(artifact), str(cartridge_root)], - detail=f"manifest DNA {manifest.dna}", - ) - - def _cache_trace_check(self, proof_root: Path) -> ProofCheck: - from .cache_fabric import CacheFabric - - layers = [layer.model_dump(mode="json") for layer in CacheFabric(self.settings, self.db).statuses()] - artifact = proof_root / "cache_trace.json" - write_json(artifact, {"layers": layers}) - valid = any(layer["entries"] > 0 for layer in layers) - return ProofCheck( - name="Cache Trace", - requirement="PRD 14.2 requires cache and memory proof with visible hit, miss, write, and replay state.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail=f"{len(layers)} cache layers recorded", - ) - - def _visual_contradiction_check(self, proof_root: Path) -> ProofCheck: - lens = VisualContradictionLens() - scenarios = build_contradiction_scenarios() - conflicts = [scenario for scenario in scenarios if scenario["expect_drift"]] - controls = [scenario for scenario in scenarios if not scenario["expect_drift"]] - detected = 0 - false_positives = 0 - categories: set[str] = set() - rows: list[dict[str, Any]] = [] - known_categories = ( - "design-token drift", - "layout drift", - "typography drift", - "copy drift", - "visual-code drift", - "temporal drift", - ) - for scenario in scenarios: - notes = lens.detect(scenario["context"]) - flagged = bool(notes) - if scenario["expect_drift"]: - if flagged: - detected += 1 - for note in notes: - label = note.split(":", 1)[0] - categories.update(category for category in known_categories if label.endswith(category)) - elif flagged: - false_positives += 1 - rows.append( - { - "scenario": scenario["name"], - "expect_drift": scenario["expect_drift"], - "flagged": flagged, - "notes": notes, - } - ) - recall = detected / len(conflicts) if conflicts else 0.0 - false_positive_rate = false_positives / len(controls) if controls else 0.0 - artifact = proof_root / "visual_contradiction_lens.json" - write_json( - artifact, - { - "scenarios": rows, - "metrics": { - "conflict_count": len(conflicts), - "control_count": len(controls), - "detected": detected, - "false_positives": false_positives, - "recall": recall, - "false_positive_rate": false_positive_rate, - "detected_categories": sorted(categories), - }, - }, - ) - valid = recall >= 0.80 and false_positive_rate == 0.0 and len(categories) >= 5 - return ProofCheck( - name="Visual Contradiction Lens", - requirement="PRD 14.2 and 14.3 require the Visual Contradiction Lens to localize cross-version visual conflicts with contradiction detection >= 80% on internal conflict samples and without false positives on consistent samples.", - status="passed" if valid else "failed", - evidence=[str(artifact)], - detail=( - f"recall {recall:.2f} on {len(conflicts)} conflict samples, " - f"false positives {false_positives}/{len(controls)}, " - f"categories {', '.join(sorted(categories))}" - ), - ) - - def _adapter_manifests( - self, - cartridge_id: str, - ) -> dict[TrainingAdapterKind, tuple[AdapterManifest, Path]]: - candidates: dict[TrainingAdapterKind, list[tuple[AdapterManifest, Path]]] = {} - for path in sorted(self.settings.adapters_dir.glob("*/adapter_manifest.json")): - manifest = AdapterManifest.model_validate_json(path.read_text(encoding="utf-8")) - candidates.setdefault(manifest.kind, []).append((manifest, path.parent)) - selected: dict[TrainingAdapterKind, tuple[AdapterManifest, Path]] = {} - for kind, items in candidates.items(): - items.sort( - key=lambda item: ( - item[0].cartridge_id == cartridge_id, - (item[1] / "adapter_model.safetensors").stat().st_mtime - if (item[1] / "adapter_model.safetensors").exists() - else 0.0, - ), - reverse=True, - ) - selected[kind] = items[0] - return selected - - def _resolve_cartridge(self, cartridge_id: str | None) -> dict[str, Any]: - rows = self.db.list_cartridges() - if not rows: - raise ValueError("No Context Cartridge is available.") - if cartridge_id is None: - return rows[0] - for row in rows: - if row["cartridge_id"] == cartridge_id: - return row - raise ValueError(f"Unknown cartridge: {cartridge_id}") - - def _proof_id(self, cartridge_id: str, dna: str | None) -> str: - raw = f"{cartridge_id}:{dna}:{secrets.token_hex(8)}" - return f"prd_proof_suite_{hashlib.sha256(raw.encode('utf-8')).hexdigest()[:16]}" - - -def render_visual_hash(item_id: str, path: Path, tile_size: int) -> None: - seed = int(hashlib.sha256(item_id.encode("utf-8")).hexdigest()[:16], 16) - rng = random.Random(seed) - image = Image.new("RGB", (tile_size, tile_size), color=(rng.randrange(256), rng.randrange(256), rng.randrange(256))) - draw = ImageDraw.Draw(image) - for _ in range(96): - x0 = rng.randrange(tile_size) - y0 = rng.randrange(tile_size) - x1 = min(tile_size, x0 + rng.randrange(4, tile_size // 3)) - y1 = min(tile_size, y0 + rng.randrange(4, tile_size // 3)) - color = (rng.randrange(256), rng.randrange(256), rng.randrange(256)) - if rng.random() < 0.5: - draw.rectangle([x0, y0, x1, y1], fill=color) - else: - draw.ellipse([x0, y0, x1, y1], fill=color) - image.save(path) - - -def render_visual_hash_arena( - source_path: Path, - slot_paths: dict[str, Path], - target_slot: str, - output_path: Path, -) -> None: - tile = 160 - canvas = Image.new("RGB", (920, 560), color=(8, 10, 14)) - draw = ImageDraw.Draw(canvas) - source_box = (72, 200, 72 + tile, 200 + tile) - slot_boxes = { - "top-left": (520, 76, 520 + tile, 76 + tile), - "top-right": (704, 76, 704 + tile, 76 + tile), - "bottom-left": (520, 324, 520 + tile, 324 + tile), - "bottom-right": (704, 324, 704 + tile, 324 + tile), - } - source_center = box_center(source_box) - target_center = box_center(slot_boxes[target_slot]) - draw.line([source_center, target_center], fill=(0, 242, 255), width=18) - draw.line([source_center, target_center], fill=(240, 255, 255), width=6) - paste_visual_hash_tile(canvas, source_path, source_box) - draw.rectangle(source_box, outline=(255, 78, 220), width=8) - for slot, path in slot_paths.items(): - box = slot_boxes[slot] - paste_visual_hash_tile(canvas, path, box) - draw.rectangle(box, outline=(180, 190, 205), width=5) - draw.ellipse( - [ - target_center[0] - 22, - target_center[1] - 22, - target_center[0] + 22, - target_center[1] + 22, - ], - outline=(255, 255, 255), - width=8, - ) - output_path.parent.mkdir(parents=True, exist_ok=True) - canvas.save(output_path) - - -def render_visual_hash_two_hop_arena( - source_path: Path, - intermediate_slot_paths: dict[str, Path], - target_slot_paths: dict[str, Path], - intermediate_slot: str, - target_slot: str, - output_path: Path, -) -> None: - tile = 136 - canvas = Image.new("RGB", (1180, 720), color=(8, 10, 14)) - draw = ImageDraw.Draw(canvas) - source_box = (64, 292, 64 + tile, 292 + tile) - intermediate_boxes = { - "top-left": (424, 104, 424 + tile, 104 + tile), - "top-right": (584, 104, 584 + tile, 104 + tile), - "bottom-left": (424, 480, 424 + tile, 480 + tile), - "bottom-right": (584, 480, 584 + tile, 480 + tile), - } - target_boxes = { - "top-left": (864, 104, 864 + tile, 104 + tile), - "top-right": (1024, 104, 1024 + tile, 104 + tile), - "bottom-left": (864, 480, 864 + tile, 480 + tile), - "bottom-right": (1024, 480, 1024 + tile, 480 + tile), - } - source_center = box_center(source_box) - intermediate_center = box_center(intermediate_boxes[intermediate_slot]) - target_center = box_center(target_boxes[target_slot]) - paste_visual_hash_tile(canvas, source_path, source_box) - draw.rectangle(source_box, outline=(255, 78, 220), width=7) - for slot, path in intermediate_slot_paths.items(): - box = intermediate_boxes[slot] - paste_visual_hash_tile(canvas, path, box) - draw.rectangle(box, outline=(140, 215, 255), width=5) - for slot, path in target_slot_paths.items(): - box = target_boxes[slot] - paste_visual_hash_tile(canvas, path, box) - draw.rectangle(box, outline=(255, 196, 92), width=5) - draw_arrow_line(draw, source_center, intermediate_center, (0, 242, 255), (240, 255, 255)) - draw_arrow_line(draw, intermediate_center, target_center, (255, 176, 36), (255, 255, 245)) - draw.ellipse( - [ - intermediate_center[0] - 26, - intermediate_center[1] - 26, - intermediate_center[0] + 26, - intermediate_center[1] + 26, - ], - outline=(0, 242, 255), - width=9, - ) - draw.ellipse( - [target_center[0] - 34, target_center[1] - 34, target_center[0] + 34, target_center[1] + 34], - outline=(255, 255, 255), - width=11, - ) - draw.ellipse( - [target_center[0] - 23, target_center[1] - 23, target_center[0] + 23, target_center[1] + 23], - outline=(255, 176, 36), - width=9, - ) - output_path.parent.mkdir(parents=True, exist_ok=True) - canvas.save(output_path) - - -def draw_arrow_line( - draw: ImageDraw.ImageDraw, - start: tuple[int, int], - end: tuple[int, int], - outer_color: tuple[int, int, int], - inner_color: tuple[int, int, int], -) -> None: - draw.line([start, end], fill=outer_color, width=16) - draw.line([start, end], fill=inner_color, width=5) - angle = math.atan2(end[1] - start[1], end[0] - start[0]) - tip = end - back = (end[0] - int(math.cos(angle) * 34), end[1] - int(math.sin(angle) * 34)) - normal = (-math.sin(angle), math.cos(angle)) - left = (back[0] + int(normal[0] * 18), back[1] + int(normal[1] * 18)) - right = (back[0] - int(normal[0] * 18), back[1] - int(normal[1] * 18)) - draw.polygon([tip, left, right], fill=outer_color) - inner_back = (end[0] - int(math.cos(angle) * 22), end[1] - int(math.sin(angle) * 22)) - inner_left = (inner_back[0] + int(normal[0] * 9), inner_back[1] + int(normal[1] * 9)) - inner_right = (inner_back[0] - int(normal[0] * 9), inner_back[1] - int(normal[1] * 9)) - draw.polygon([tip, inner_left, inner_right], fill=inner_color) - - -def paste_visual_hash_tile(canvas: Image.Image, path: Path, box: tuple[int, int, int, int]) -> None: - tile = Image.open(path).convert("RGB").resize((box[2] - box[0], box[3] - box[1]), Image.Resampling.LANCZOS) - canvas.paste(tile, box[:2]) - - -def box_center(box: tuple[int, int, int, int]) -> tuple[int, int]: - return ((box[0] + box[2]) // 2, (box[1] + box[3]) // 2) - - -def count_safetensor_keys(path: Path) -> int: - return int(safetensor_weight_stats(path)["tensor_count"]) - - -def peft_config_loads(path: Path) -> bool: - try: - from peft import PeftConfig - - PeftConfig.from_pretrained(path) - return True - except Exception: # noqa: BLE001 # a probe records its failure instead of crashing the proof run - return False - - -def validate_training_contract(manifest: AdapterManifest) -> dict[str, Any]: - target_modules_ok = training_target_modules_ok(manifest.kind, manifest.target_modules) - report = { - "proof_version_ok": manifest.proof_version >= TRAINING_PROOF_VERSION, - "method_ok": manifest.training_method == TRAINING_METHOD, - "precision_ok": manifest.precision == TRAINING_PRECISION and manifest.bf16, - "framework_ok": manifest.framework == TRAINING_FRAMEWORK, - "optimizer_ok": manifest.optimizer == "paged_adamw_8bit", - "optimizer_family_ok": manifest.optimizer_family == "8-bit AdamW / paged optimizer", - "gradient_checkpointing_ok": ( - manifest.gradient_checkpointing - and manifest.gradient_checkpointing_mode == GRADIENT_CHECKPOINTING_MODE - ), - "autocast_dtype_ok": manifest.autocast_dtype == AUTOCAST_DTYPE, - "unsloth_version_ok": bool(manifest.unsloth_version), - "triton_version_ok": bool(manifest.triton_version), - "xformers_version_ok": bool(manifest.xformers_version), - "unsloth_model_type_ok": bool(manifest.unsloth_model_type), - "max_length_ok": manifest.max_length >= 8192, - "target_modules_ok": target_modules_ok, - } - report["valid"] = all(report.values()) - return report - - -def training_target_modules_ok(kind: TrainingAdapterKind, target_modules: list[str]) -> bool: - targets = set(target_modules) - core = {"q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"} - if kind == TrainingAdapterKind.hme_core_lora: - return core.issubset(targets) - if kind == TrainingAdapterKind.hme_vision_lora: - return bool(targets) and any("visual" in target or "vision" in target for target in targets) - if kind == TrainingAdapterKind.hme_ttt_vl: - has_visual = any("visual" in target or "vision" in target for target in targets) - has_mlp = bool({"gate_proj", "up_proj", "down_proj"} & targets) - return has_visual and has_mlp - if kind == TrainingAdapterKind.hme_visual_mtp: - return any("mtp" in target or "draft" in target or target == "lm_head" for target in targets) - if kind in {TrainingAdapterKind.hme_router, TrainingAdapterKind.hme_retrieval}: - return core.issubset(targets) - return False - - -def training_payload_contract_ok(payload: dict[str, Any], manifest: AdapterManifest) -> bool: - return ( - int(payload.get("proof_version") or 0) >= TRAINING_PROOF_VERSION - and payload.get("training_method") == TRAINING_METHOD - and payload.get("precision") == TRAINING_PRECISION - and payload.get("framework") == TRAINING_FRAMEWORK - and payload.get("optimizer_family") == manifest.optimizer_family - and bool(payload.get("gradient_checkpointing")) - and payload.get("gradient_checkpointing_mode") == GRADIENT_CHECKPOINTING_MODE - and payload.get("autocast_dtype") == AUTOCAST_DTYPE - and payload.get("unsloth_version") == manifest.unsloth_version - and payload.get("triton_version") == manifest.triton_version - and payload.get("xformers_version") == manifest.xformers_version - and payload.get("unsloth_model_type") == manifest.unsloth_model_type - ) - - -def validate_training_trace(manifest: AdapterManifest, adapter_root: Path) -> dict[str, Any]: - trace_path = adapter_root / "training_trace.jsonl" - summary_path = adapter_root / "training_summary.json" - report: dict[str, Any] = { - "trace_path": str(trace_path), - "summary_path": str(summary_path), - "trace_exists": trace_path.exists(), - "summary_exists": summary_path.exists(), - "trace_sha256_ok": False, - "summary_sha256_ok": False, - "step_records": 0, - "loss_history_records": len(manifest.loss_history), - "loss_history_ok": False, - "summary_matches_manifest": False, - "summary_contract_ok": False, - "run_start_contract_ok": False, - "run_finish_contract_ok": False, - "cuda_training": False, - "bf16_supported": False, - "bf16": manifest.bf16, - "valid": False, - } - if not trace_path.exists() or not summary_path.exists(): - return report - try: - trace_rows = [ - json.loads(line) - for line in trace_path.read_text(encoding="utf-8").splitlines() - if line.strip() - ] - summary = json.loads(summary_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return report - - step_rows = [row for row in trace_rows if row.get("event") == "step"] - start_row = next((row for row in trace_rows if row.get("event") == "run_start"), {}) - finish_row = next((row for row in reversed(trace_rows) if row.get("event") == "run_finish"), {}) - losses = [row.get("loss") for row in step_rows] - finite_losses = all(isinstance(loss, (int, float)) and math.isfinite(float(loss)) for loss in losses) - final_loss_ok = ( - manifest.final_loss is not None - and bool(losses) - and abs(float(losses[-1]) - float(manifest.final_loss)) < 1e-6 - ) - report.update( - { - "trace_sha256_ok": ( - manifest.training_trace_sha256 is not None - and sha256_file(trace_path) == manifest.training_trace_sha256 - ), - "summary_sha256_ok": ( - manifest.training_summary_sha256 is not None - and sha256_file(summary_path) == manifest.training_summary_sha256 - ), - "step_records": len(step_rows), - "loss_history_ok": ( - len(manifest.loss_history) == manifest.max_steps - and finite_losses - and final_loss_ok - ), - "summary_matches_manifest": ( - summary.get("adapter_kind") == manifest.kind.value - and summary.get("base_model") == manifest.base_model - and summary.get("dataset_sha256") == manifest.dataset_sha256 - and int(summary.get("max_steps") or -1) == manifest.max_steps - and int(summary.get("trainable_params") or -1) == manifest.trainable_params - ), - "summary_contract_ok": training_payload_contract_ok(summary, manifest), - "run_start_contract_ok": training_payload_contract_ok(start_row, manifest), - "run_finish_contract_ok": training_payload_contract_ok(finish_row, manifest), - } - ) - environment = summary.get("environment") if isinstance(summary.get("environment"), dict) else {} - device_name = str(environment.get("device_name") or "") - report["cuda_training"] = bool(environment.get("cuda_available")) and "RTX 4090" in device_name - report["bf16_supported"] = bool(environment.get("bf16_supported")) - report["valid"] = ( - report["trace_sha256_ok"] - and report["summary_sha256_ok"] - and report["step_records"] == manifest.max_steps - and report["loss_history_ok"] - and report["summary_matches_manifest"] - and report["summary_contract_ok"] - and report["run_start_contract_ok"] - and report["run_finish_contract_ok"] - and report["cuda_training"] - and report["bf16_supported"] - and manifest.bf16 - ) - return report - - -def validate_training_proof(manifest: AdapterManifest, adapter_root: Path) -> dict[str, Any]: - proof_path = adapter_root / "training_proof.json" - report: dict[str, Any] = { - "proof_path": str(proof_path), - "proof_exists": proof_path.exists(), - "proof_sha256_ok": False, - "hash_changed": False, - "optimizer_update_count": manifest.optimizer_update_count, - "optimizer_update_count_ok": False, - "max_optimizer_update_l2": manifest.max_optimizer_update_l2, - "max_optimizer_update_l2_ok": False, - "loss_delta": manifest.loss_delta, - "loss_delta_finite": False, - "updates_have_gradients": False, - "updates_have_weight_delta": False, - "proof_weight_sha256_ok": False, - "proof_weight_stats_ok": False, - "proof_cuda_bf16": False, - "proof_bf16_supported": False, - "proof_contract_ok": False, - "proof_optimizer_ok": False, - "proof_max_length_ok": False, - "proof_target_modules_ok": False, - "proof_unsloth_runtime_ok": False, - "proof_matches_manifest": False, - "valid": False, - } - if not proof_path.exists(): - return report - try: - proof = json.loads(proof_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return report - - updates = proof.get("optimizer_updates") if isinstance(proof.get("optimizer_updates"), list) else [] - update_l2_values = [ - float(item.get("update_l2")) - for item in updates - if isinstance(item, dict) and isinstance(item.get("update_l2"), (int, float)) - ] - gradient_l2_values = [ - float(item.get("gradient_l2")) - for item in updates - if isinstance(item, dict) and isinstance(item.get("gradient_l2"), (int, float)) - ] - weight_report = safetensor_weight_stats(adapter_root / "adapter_model.safetensors") - proof_weight_stats = proof.get("adapter_weight_stats") if isinstance(proof.get("adapter_weight_stats"), dict) else {} - proof_environment = proof.get("environment") if isinstance(proof.get("environment"), dict) else {} - report.update( - { - "proof_sha256_ok": ( - manifest.training_proof_sha256 is not None - and sha256_file(proof_path) == manifest.training_proof_sha256 - ), - "hash_changed": ( - bool(manifest.initial_trainable_sha256) - and bool(manifest.final_trainable_sha256) - and manifest.initial_trainable_sha256 != manifest.final_trainable_sha256 - ), - "optimizer_update_count_ok": ( - manifest.optimizer_update_count > 0 - and len(updates) == manifest.optimizer_update_count - ), - "max_optimizer_update_l2_ok": ( - manifest.max_optimizer_update_l2 is not None - and math.isfinite(float(manifest.max_optimizer_update_l2)) - and float(manifest.max_optimizer_update_l2) > 0.0 - ), - "loss_delta_finite": ( - manifest.loss_delta is not None - and math.isfinite(float(manifest.loss_delta)) - ), - "updates_have_gradients": bool(gradient_l2_values) - and all(math.isfinite(value) and value > 0.0 for value in gradient_l2_values), - "updates_have_weight_delta": bool(update_l2_values) - and all(math.isfinite(value) and value > 0.0 for value in update_l2_values), - "proof_weight_sha256_ok": ( - isinstance(proof.get("adapter_weight_sha256"), str) - and proof.get("adapter_weight_sha256") == weight_report["sha256"] - ), - "proof_weight_stats_ok": ( - proof_weight_stats.get("sha256") == weight_report["sha256"] - and proof_weight_stats.get("tensor_count") == weight_report["tensor_count"] - and proof_weight_stats.get("lora_tensor_count") == weight_report["lora_tensor_count"] - and proof_weight_stats.get("total_elements") == weight_report["total_elements"] - and proof_weight_stats.get("nonzero_elements") == weight_report["nonzero_elements"] - and int(proof_weight_stats.get("lora_tensor_count") or 0) > 0 - and int(proof_weight_stats.get("nonzero_elements") or 0) > 0 - and bool(proof_weight_stats.get("all_finite")) - ), - "proof_cuda_bf16": ( - bool(proof.get("bf16")) - and bool(proof_environment.get("cuda_available")) - and "RTX 4090" in str(proof_environment.get("device_name") or "") - ), - "proof_bf16_supported": bool(proof_environment.get("bf16_supported")), - "proof_contract_ok": training_payload_contract_ok(proof, manifest), - "proof_optimizer_ok": proof.get("optimizer") == "paged_adamw_8bit", - "proof_max_length_ok": int(proof.get("max_length") or 0) >= 8192, - "proof_target_modules_ok": training_target_modules_ok( - manifest.kind, - proof.get("target_modules") if isinstance(proof.get("target_modules"), list) else [], - ), - "proof_unsloth_runtime_ok": ( - bool(proof.get("unsloth_version")) - and bool(proof.get("triton_version")) - and bool(proof.get("xformers_version")) - and bool(proof.get("unsloth_model_type")) - ), - "proof_matches_manifest": ( - proof.get("adapter_kind") == manifest.kind.value - and proof.get("proof_version") == manifest.proof_version - and proof.get("training_method") == manifest.training_method - and proof.get("precision") == manifest.precision - and proof.get("framework") == manifest.framework - and proof.get("base_model") == manifest.base_model - and proof.get("dataset_sha256") == manifest.dataset_sha256 - and proof.get("sample_count") == manifest.sample_count - and proof.get("max_steps") == manifest.max_steps - and proof.get("max_length") == manifest.max_length - and proof.get("rank") == manifest.rank - and proof.get("alpha") == manifest.alpha - and proof.get("target_modules") == manifest.target_modules - and proof.get("optimizer") == manifest.optimizer - and proof.get("optimizer_family") == manifest.optimizer_family - and proof.get("learning_rate") == manifest.learning_rate - and proof.get("gradient_accumulation_steps") == manifest.gradient_accumulation_steps - and proof.get("gradient_checkpointing") == manifest.gradient_checkpointing - and proof.get("gradient_checkpointing_mode") == manifest.gradient_checkpointing_mode - and proof.get("autocast_dtype") == manifest.autocast_dtype - and proof.get("unsloth_version") == manifest.unsloth_version - and proof.get("triton_version") == manifest.triton_version - and proof.get("xformers_version") == manifest.xformers_version - and proof.get("unsloth_model_type") == manifest.unsloth_model_type - and proof.get("bf16") == manifest.bf16 - and proof.get("trainable_params") == manifest.trainable_params - and proof.get("total_params") == manifest.total_params - and proof.get("initial_trainable_sha256") == manifest.initial_trainable_sha256 - and proof.get("final_trainable_sha256") == manifest.final_trainable_sha256 - and proof.get("optimizer_update_count") == manifest.optimizer_update_count - and proof.get("max_optimizer_update_l2") == manifest.max_optimizer_update_l2 - ), - } - ) - report["valid"] = ( - report["proof_sha256_ok"] - and report["hash_changed"] - and report["optimizer_update_count_ok"] - and report["max_optimizer_update_l2_ok"] - and report["loss_delta_finite"] - and report["updates_have_gradients"] - and report["updates_have_weight_delta"] - and report["proof_weight_sha256_ok"] - and report["proof_weight_stats_ok"] - and report["proof_cuda_bf16"] - and report["proof_bf16_supported"] - and report["proof_contract_ok"] - and report["proof_optimizer_ok"] - and report["proof_max_length_ok"] - and report["proof_target_modules_ok"] - and report["proof_unsloth_runtime_ok"] - and report["proof_matches_manifest"] - ) - return report - - -def parquet_row_count(path: Path) -> int: - if not path.exists(): - return -1 - import pyarrow.parquet as pq - - return pq.read_metadata(path).num_rows - - -def compiled_context_digest(context: Any) -> str: - payload = { - "hits": [hit.atom.atom_id for hit in context.hits], - "packed_text": context.packed_text, - "token_equivalent": context.token_equivalent, - } - data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") - return hashlib.sha256(data).hexdigest() - - -def read_memory_capsule(path: Path) -> dict[str, Any]: - data = path.read_bytes() - prefix = b"ELYMEMCAP1\n" - if not data.startswith(prefix): - raise ValueError(f"Invalid memory capsule header: {path}") - try: - payload = zstd.ZstdDecompressor().decompress(data[len(prefix) :]) - parsed = json.loads(payload.decode("utf-8")) - except (zstd.ZstdError, UnicodeDecodeError, json.JSONDecodeError) as error: - raise ValueError(f"Cannot decode the memory capsule at {path}: {error}") from error - if not isinstance(parsed, dict): - raise ValueError(f"Invalid memory capsule payload: {path}") # noqa: TRY004 # the capsule's own bytes are wrong, not the caller's argument - return parsed - - -def validated_segment_hash(segment: dict[str, Any]) -> str: - expected = segment.get("segment_sha256") - if not isinstance(expected, str): - return "" - payload = {key: value for key, value in segment.items() if key != "segment_sha256"} - actual = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() - return actual if actual == expected else "" - - -def hashhop_chain_valid(proof: HashHopProof) -> bool: - chain_artifacts = [Path(path) for path in proof.artifacts if path.endswith(".jsonl")] - if len(chain_artifacts) != 1 or not chain_artifacts[0].exists(): - return False - try: - rows = [ - json.loads(line) - for line in chain_artifacts[0].read_text(encoding="utf-8").splitlines() - if line.strip() - ] - except (OSError, json.JSONDecodeError): - return False # an unreadable chain is not a valid chain, and this is a predicate - if len(rows) != proof.hops: - return False - if rows[0]["from"] != proof.query_id or rows[-1]["to"] != proof.expected_target_id: - return False - for index in range(1, len(rows)): - if rows[index - 1]["to"] != rows[index]["from"]: - return False - return all(Path(path).exists() for path in proof.artifacts) - - -def extract_hashhop_target(text: str) -> str | None: - stripped = text.strip() - try: - payload = json.loads(stripped) - except json.JSONDecodeError: - payload = extract_json_object(stripped) - if isinstance(payload, dict): - for key in ("answer", "target", "target_id", "final_target_id"): - value = payload.get(key) - if isinstance(value, str): - candidate = normalize_hash_id(value) - if candidate: - return candidate - match = re.search(r"\b[a-fA-F0-9]{32}\b", stripped) - return match.group(0).lower() if match else None - - -def extract_visual_target_slot(text: str) -> str | None: - stripped = text.strip() - payload: dict[str, Any] | None - try: - parsed = json.loads(stripped) - payload = parsed if isinstance(parsed, dict) else None - except json.JSONDecodeError: - payload = extract_json_object(stripped) - if isinstance(payload, dict): - for key in ("target_slot", "final_slot", "slot", "answer"): - value = payload.get(key) - if isinstance(value, str): - slot = normalize_visual_slot(value) - if slot: - return slot - return normalize_visual_slot(stripped) - - -def extract_visual_intermediate_slot(text: str) -> str | None: - stripped = text.strip() - try: - parsed = json.loads(stripped) - payload = parsed if isinstance(parsed, dict) else None - except json.JSONDecodeError: - payload = extract_json_object(stripped) - if isinstance(payload, dict): - for key in ("intermediate_slot", "middle_slot", "first_hop_slot"): - value = payload.get(key) - if isinstance(value, str): - slot = normalize_visual_slot(value) - if slot: - return slot - return None - - -def extract_json_object(text: str) -> dict[str, Any] | None: - start = text.find("{") - end = text.rfind("}") - if start < 0 or end <= start: - return None - try: - payload = json.loads(text[start : end + 1]) - except json.JSONDecodeError: - return None - return payload if isinstance(payload, dict) else None - - -def normalize_hash_id(value: str) -> str | None: - match = re.search(r"\b[a-fA-F0-9]{32}\b", value) - return match.group(0).lower() if match else None - - -def normalize_visual_slot(value: str) -> str | None: - normalized = re.sub(r"[^a-z]+", "-", value.lower()).strip("-") - aliases = { - "top-left": "top-left", - "upper-left": "top-left", - "left-top": "top-left", - "top-right": "top-right", - "upper-right": "top-right", - "right-top": "top-right", - "bottom-left": "bottom-left", - "lower-left": "bottom-left", - "left-bottom": "bottom-left", - "bottom-right": "bottom-right", - "lower-right": "bottom-right", - "right-bottom": "bottom-right", - } - if normalized in aliases: - return aliases[normalized] - for alias, slot in aliases.items(): - if re.search(rf"\b{re.escape(alias)}\b", normalized): - return slot - return None - - -def copy_proof_artifacts(proof: HashHopProof, target: Path) -> None: - target.mkdir(parents=True, exist_ok=True) - source_roots = {Path(path).parent for path in proof.artifacts} - for source_root in source_roots: - if source_root.name == "visual_blocks": - shutil.copytree(source_root, target / "visual_blocks", dirs_exist_ok=True) - for artifact in proof.artifacts: - path = Path(artifact) - if path.is_file() and path.parent.name != "visual_blocks": - shutil.copy2(path, target / path.name) - (target / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") - - -def contradiction_atom( - atom_id: str, - source_id: str, - source: str, - text: str, - modality: Modality, - source_kind: str, - version: str | None = None, -) -> EvidenceAtom: - metadata: dict[str, Any] = {"source_kind": source_kind} - if version is not None: - metadata["version"] = version - return EvidenceAtom( - atom_id=atom_id, - modality=modality, - source_id=source_id, - source=source, - text=text, - trust=TrustScores(parser=0.95, ocr=0.9, model=0.85), - token_equivalent=max(1, len(text) // 4), - metadata=metadata, - ) - - -def contradiction_context(question: str, atoms: list[EvidenceAtom]) -> CompiledContext: - hits = [ - RetrievalHit(atom=atom, sparse_score=0.8, dense_score=0.0, graph_score=0.0, final_score=0.8) - for atom in atoms - ] - plan = ContextPlan( - question=question, - profile=RuntimeProfile.research_theater, - required_modalities=sorted({atom.modality for atom in atoms}), - evidence_budget_atoms=len(atoms), - token_budget=sum(atom.token_equivalent for atom in atoms), - retrieval_query=question, - compression_strategy="visual-contradiction-eval", - verifier_contract=["Report contradictions as contradiction_notes."], - ) - packed = "\n".join(f"[{atom.atom_id}] {atom.text}" for atom in atoms) - return CompiledContext( - plan=plan, - hits=hits, - packed_text=packed, - token_equivalent=sum(atom.token_equivalent for atom in atoms), - cache_trace_id="visual_contradiction_eval", - ) - - -def build_contradiction_scenarios() -> list[dict[str, Any]]: - """Deterministic cross-version conflict and control samples (PRD 11.15). - - Each conflict pair injects a single genuine drift across two distinct - sources; each control pair is internally consistent or scoped to different - components. The samples exercise the live ``VisualContradictionLens`` so the - proof measures real recall and false-positive rate. - """ - - scenarios: list[dict[str, Any]] = [] - - def conflict(name: str, atoms: list[EvidenceAtom]) -> None: - scenarios.append({"name": name, "context": contradiction_context(name, atoms), "expect_drift": True}) - - def control(name: str, atoms: list[EvidenceAtom]) -> None: - scenarios.append({"name": name, "context": contradiction_context(name, atoms), "expect_drift": False}) - - conflict( - "button-radius-token-drift", - [ - contradiction_atom("home_btn_v4", "home_v4", "home_v4.figma", "button.radius: 8px", Modality.ui_screenshot, "figma"), - contradiction_atom("checkout_btn_v4", "checkout_v4", "checkout_v4.figma", "button.radius: 12px", Modality.ui_screenshot, "figma"), - ], - ) - conflict( - "primary-color-token-drift", - [ - contradiction_atom("spec_color", "brand_spec", "brand_guide.pdf", "primary.color: #2563eb", Modality.pdf_page, "spec"), - contradiction_atom("impl_color", "theme_css", "src/theme.css", "primary.color: #1d4ed8", Modality.code, "css"), - ], - ) - conflict( - "heading-typography-drift", - [ - contradiction_atom("home_heading", "home_shot", "home_v3.png", "heading.font-size: 24px", Modality.ui_screenshot, "ui_screenshot"), - contradiction_atom("checkout_heading", "checkout_shot", "checkout_v3.png", "heading.font-size: 28px", Modality.ui_screenshot, "ui_screenshot"), - ], - ) - conflict( - "card-spacing-layout-drift", - [ - contradiction_atom("card_home", "card_home_shot", "home_v3.png", "card.padding: 16px", Modality.ui_screenshot, "ui_screenshot"), - contradiction_atom("card_settings", "card_settings_shot", "settings_v3.png", "card.padding: 24px", Modality.ui_screenshot, "ui_screenshot"), - ], - ) - conflict( - "cta-copy-drift", - [ - contradiction_atom("cta_v1", "shot_v1", "cart_v1.png", 'cta.label: "Checkout"', Modality.ui_screenshot, "ui_screenshot"), - contradiction_atom("cta_v4", "shot_v4", "cart_v4.png", 'cta.label: "Pay now"', Modality.ui_screenshot, "ui_screenshot"), - ], - ) - conflict( - "banner-radius-temporal-drift", - [ - contradiction_atom("banner_v1", "banner_spec_v1", "banner_v1.figma", "banner.radius: 4px", Modality.ui_screenshot, "figma", version="v1"), - contradiction_atom("banner_v4", "banner_spec_v4", "banner_v4.figma", "banner.radius: 10px", Modality.ui_screenshot, "figma", version="v4"), - ], - ) - conflict( - "toggle-accent-visual-code-drift", - [ - contradiction_atom("toggle_design", "toggle_figma", "toggle.figma", "toggle.color: #10b981", Modality.ui_screenshot, "figma"), - contradiction_atom("toggle_code", "toggle_css", "src/toggle.css", "toggle.color: #059669", Modality.code, "css"), - ], - ) - - control( - "consistent-button-radius", - [ - contradiction_atom("ctrl_btn_spec", "ctrl_spec", "spec.pdf", "button.radius: 8px", Modality.pdf_page, "spec"), - contradiction_atom("ctrl_btn_css", "ctrl_css", "src/tokens.css", "button.radius = 8px", Modality.code, "css"), - ], - ) - control( - "distinct-components-not-a-conflict", - [ - contradiction_atom("ctrl_button_radius", "ctrl_a", "a.css", "button.radius: 8px", Modality.code, "css"), - contradiction_atom("ctrl_input_radius", "ctrl_b", "b.css", "input.radius: 12px", Modality.code, "css"), - ], - ) - control( - "single-source-multiple-rules", - [ - contradiction_atom( - "ctrl_single_css", - "ctrl_single", - "src/base.css", - "header.padding: 8px; footer.padding: 24px; aside.padding: 16px", - Modality.code, - "css", - ), - ], - ) - control( - "consistent-primary-color", - [ - contradiction_atom("ctrl_color_spec", "ctrl_color_a", "brand.pdf", "primary.color: #2563eb", Modality.pdf_page, "spec"), - contradiction_atom("ctrl_color_css", "ctrl_color_b", "theme.css", "primary.color: #2563eb", Modality.code, "css"), - ], - ) - control( - "prose-without-design-facts", - [ - contradiction_atom("ctrl_prose_a", "ctrl_prose_1", "notes_a.txt", "The onboarding flow feels smooth and the colors look balanced.", Modality.text, "text"), - contradiction_atom("ctrl_prose_b", "ctrl_prose_2", "notes_b.txt", "Users reported the checkout button is easy to find.", Modality.text, "text"), - ], - ) - - return scenarios - - -def write_json(path: Path, payload: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") - - -def read_json_or_empty(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - value = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - return value if isinstance(value, dict) else {} - - -def format_scores(scores: dict[str, Any]) -> str: - parts: list[str] = [] - for name in sorted(scores): - score = scores[name] - if isinstance(score, (int, float)) and math.isfinite(float(score)): - parts.append(f"{name}={float(score):.2f}") - else: - parts.append(f"{name}=unscored") - return ", ".join(parts) or "none" - - -def research_module_checks(report: dict[str, Any]) -> list[dict[str, Any]]: - checks: list[dict[str, Any]] = [] - - def add(module: str, passed: bool, detail: str) -> None: - checks.append({"module": module, "passed": passed, "detail": detail}) - - v_nsa = report.get("v_nsa") if isinstance(report.get("v_nsa"), dict) else {} - sparse_blocks = v_nsa.get("sparse_blocks") if isinstance(v_nsa.get("sparse_blocks"), list) else [] - add( - "v_nsa", - int(v_nsa.get("sparse_block_count") or 0) > 0 and bool(sparse_blocks), - f"{len(sparse_blocks)} sparse blocks", - ) - - dvkv = report.get("dvkv") if isinstance(report.get("dvkv"), dict) else {} - selected = dvkv.get("selected") if isinstance(dvkv.get("selected"), list) else [] - candidate_tokens = int(dvkv.get("candidate_tokens") or 0) - budget = int(dvkv.get("budget") or 0) - dvkv_scores_valid = all( - is_finite_number(item.get("attention_importance")) and is_finite_number(item.get("evidence_value")) - for item in selected - if isinstance(item, dict) - ) - add( - "dvkv", - candidate_tokens > 0 and 0 < len(selected) <= budget and dvkv_scores_valid, - f"{len(selected)}/{candidate_tokens} visual tokens selected", - ) - - x_modal = report.get("x_modal_kv") if isinstance(report.get("x_modal_kv"), dict) else {} - shared_blocks = x_modal.get("shared_blocks") if isinstance(x_modal.get("shared_blocks"), list) else [] - x_modal_valid = bool(shared_blocks) and all( - isinstance(block, dict) and bool(block.get("block_id")) and bool(block.get("atom_ids")) - for block in shared_blocks - ) - add("x_modal_kv", x_modal_valid, f"{len(shared_blocks)} shared KV blocks") - - prefetch = report.get("speculative_prefetch") if isinstance(report.get("speculative_prefetch"), dict) else {} - prefetch_blocks = prefetch.get("prefetch_blocks") if isinstance(prefetch.get("prefetch_blocks"), list) else [] - priorities = [ - float(block.get("priority")) - for block in prefetch_blocks - if isinstance(block, dict) and is_finite_number(block.get("priority")) - ] - prefetch_valid = ( - bool(prefetch_blocks) - and len(priorities) == len(prefetch_blocks) - and priorities == sorted(priorities, reverse=True) - and all(int(block.get("bytes_estimate") or 0) > 0 for block in prefetch_blocks if isinstance(block, dict)) - ) - add("speculative_prefetch", prefetch_valid, f"{len(prefetch_blocks)} prefetch blocks") - - eviction = report.get("predictive_eviction") if isinstance(report.get("predictive_eviction"), dict) else {} - layers = eviction.get("layers") if isinstance(eviction.get("layers"), list) else [] - eviction_valid = bool(layers) and all( - isinstance(layer, dict) - and bool(layer.get("replay_token")) - and is_finite_number(layer.get("hit_rate")) - and int(layer.get("entries") or 0) >= 0 - for layer in layers - ) - add("predictive_eviction", eviction_valid, f"{len(layers)} cache layers") - - retrieval = report.get("differentiable_retrieval") if isinstance(report.get("differentiable_retrieval"), dict) else {} - probabilities = retrieval.get("probabilities") if isinstance(retrieval.get("probabilities"), list) else [] - probability_values = [ - float(item.get("probability")) - for item in probabilities - if isinstance(item, dict) and is_finite_number(item.get("probability")) - ] - probability_sum = sum(probability_values) - retrieval_valid = ( - bool(probabilities) - and len(probability_values) == len(probabilities) - and abs(probability_sum - 1.0) <= 1e-6 - and bool(retrieval.get("straight_through_atom_id")) - ) - add("differentiable_retrieval", retrieval_valid, f"probability sum {probability_sum:.6f}") - - graph = report.get("evidence_graph") if isinstance(report.get("evidence_graph"), dict) else {} - relations = graph.get("relations") if isinstance(graph.get("relations"), list) else [] - edge_count = int(graph.get("edge_count_sample") or 0) - add("evidence_graph", edge_count > 0 and "cites" in relations, f"{edge_count} sampled edges") - - sleep = report.get("sleep_consolidation") if isinstance(report.get("sleep_consolidation"), dict) else {} - durable = sleep.get("durable_candidates") if isinstance(sleep.get("durable_candidates"), list) else [] - add( - "sleep_consolidation", - int(sleep.get("candidate_count") or 0) > 0 and bool(durable) and bool(sleep.get("target_adapter")), - f"{len(durable)} durable candidates", - ) - - return checks - - -def is_finite_number(value: Any) -> bool: - return isinstance(value, (int, float)) and math.isfinite(float(value)) - - -def project_root() -> Path: - current = Path.cwd().resolve() - for candidate in (current, *current.parents): - if (candidate / "PRD.md").exists() and (candidate / "backend" / "src" / "ely_eye").exists(): - return candidate - return current - - -def discover_repository_files(root: Path) -> list[Path]: - ignored_dirs = {".git", ".ely_eye", ".venv", ".venv-linux", "__pycache__", "node_modules", "dist", "build", "target"} - files: list[Path] = [] - for current, dirs, names in os.walk(root): - dirs[:] = [name for name in dirs if name not in ignored_dirs and not name.startswith(".venv")] - current_path = Path(current) - for name in names: - path = current_path / name - try: - relative = path.relative_to(root) - except ValueError: - continue - if any(part in ignored_dirs or part.startswith(".venv") for part in relative.parts): - continue - try: - path.stat() - except OSError: - continue - files.append(path) - return files - - -def pass_rate(proofs: list[HashHopProof]) -> float: - if not proofs: - return 0.0 - return sum(1 for proof in proofs if proof.passed) / len(proofs) - - -def collect_model_visual_proofs( - evaluator: HashHopEvaluator, - hops: int, - token_equivalent: int, - min_samples: int, - target_rate: float, - max_samples: int, -) -> list[HashHopProof]: - proofs: list[HashHopProof] = [] - while len(proofs) < max_samples: - proofs.append(evaluator.generate_model_visual_proof(hops, token_equivalent, max_proofs=5)) - if len(proofs) >= min_samples and pass_rate(proofs) >= target_rate: - break - return proofs - - -def compact_generation_context(context: Any, section_hint: str, max_chars: int = 2400) -> Any: - text_hits = [hit for hit in context.hits if hit.atom.modality.value == "text"] - hits = (text_hits or context.hits)[:1] - sections: list[str] = [] - for hit in hits: - atom = hit.atom - evidence_text = extract_markdown_section(atom.text, section_hint) or atom.text - evidence_text = evidence_text.strip()[:max_chars] - citation = { - "atom_id": atom.atom_id, - "modality": atom.modality.value, - "source": atom.source, - "image_ref": atom.image_ref, - "layout": atom.layout.model_dump() if atom.layout else None, - "score": hit.final_score, - } - sections.append(f"[EVIDENCE]\n{citation}\n{evidence_text}\n[/EVIDENCE]") - packed_text = "\n\n".join(sections) - return context.model_copy( - update={ - "hits": hits, - "packed_text": packed_text, - "token_equivalent": max(1, len(packed_text) // 4), - } - ) - - -def extract_markdown_section(text: str, section_hint: str) -> str: - lines = text.splitlines() - start = None - for index, line in enumerate(lines): - stripped = line.strip() - if stripped.startswith("#") and section_hint in stripped: - start = index - break - if start is None: - return "" - end = len(lines) - start_level = len(lines[start]) - len(lines[start].lstrip("#")) - for index in range(start + 1, len(lines)): - stripped = lines[index].strip() - if stripped.startswith("#"): - level = len(lines[index]) - len(lines[index].lstrip("#")) - if level <= start_level: - end = index - break - return "\n".join(lines[start:end]) - - -def relative_artifacts(root: Path) -> dict[str, str]: - artifacts: dict[str, str] = {} - for path in sorted(root.rglob("*")): - if path.is_file(): - key = path.relative_to(root).as_posix() - artifacts[key] = key - return artifacts diff --git a/backend/src/ely_eye/evaluation_schemas.py b/backend/src/ely_eye/evaluation_schemas.py index d2a624a..b72abe2 100644 --- a/backend/src/ely_eye/evaluation_schemas.py +++ b/backend/src/ely_eye/evaluation_schemas.py @@ -105,16 +105,26 @@ class HashHopProof(BaseModel): created_at: datetime = Field(default_factory=utc_now) class ProofCheck(BaseModel): + """One claim, one verdict. + + `deferred` means the check is written and runnable but needs something this machine does not + have (model weights, a second source). `removed` means the old check proved nothing and the + claim now waits for a check that can. Both carry a reason and the command that runs the check. + """ + name: str requirement: str - status: Literal["passed", "failed"] + status: Literal["passed", "failed", "deferred", "removed"] evidence: list[str] detail: str + reason: str = "" + rerun: str = "" + class ProofSuiteReport(BaseModel): proof_id: str cartridge_id: str - status: Literal["passed", "failed"] + status: Literal["passed", "failed", "deferred"] created_at: datetime = Field(default_factory=utc_now) checks: list[ProofCheck] artifacts: dict[str, str] diff --git a/backend/src/ely_eye/profiles.py b/backend/src/ely_eye/profiles.py index f91fe9a..a44e1b0 100644 --- a/backend/src/ely_eye/profiles.py +++ b/backend/src/ely_eye/profiles.py @@ -410,10 +410,12 @@ def resolve_llama_server() -> str | None: def windows_path_to_wsl(path: Path) -> str: + """Map a Windows path to the WSL mount a WSL shell can cd into; a POSIX path is already correct.""" resolved = path.resolve() drive = resolved.drive.rstrip(":").lower() - parts = resolved.parts[1:] - return "/mnt/" + drive + "/" + "/".join(parts) + if not drive: + return str(resolved) + return "/mnt/" + drive + "/" + "/".join(resolved.parts[1:]) def wsl_quote(value: str) -> str: diff --git a/backend/src/ely_eye/proofs/__init__.py b/backend/src/ely_eye/proofs/__init__.py new file mode 100644 index 0000000..1b4266b --- /dev/null +++ b/backend/src/ely_eye/proofs/__init__.py @@ -0,0 +1,6 @@ +"""Proof machinery: one module per kind of claim, one verdict per check.""" + +from .hashhop import HashHopEvaluator +from .suite import PRDProofSuite + +__all__ = ["HashHopEvaluator", "PRDProofSuite"] diff --git a/backend/src/ely_eye/proofs/arena.py b/backend/src/ely_eye/proofs/arena.py new file mode 100644 index 0000000..a1f7a7a --- /dev/null +++ b/backend/src/ely_eye/proofs/arena.py @@ -0,0 +1,224 @@ +"""Draw the HashHop puzzles and read the model's answer back out of them.""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +from pathlib import Path +from typing import Any, Literal + +from PIL import Image, ImageDraw, ImageFont + +# The four arena positions a model may answer with. Naming them once keeps the puzzle, the parser, +# and the scoring keyed on the same values instead of four spellings in three files. +SlotName = Literal["top-left", "top-right", "bottom-left", "bottom-right"] +SLOTS: tuple[SlotName, ...] = ("top-left", "top-right", "bottom-left", "bottom-right") + +ID_LINE_CHARS = 8 +CANVAS_BACKGROUND = (8, 10, 14) + + +def render_visual_hash(item_id: str, path: Path, tile_size: int) -> None: + """Print the id on the tile, over a pattern derived from the same id. + + A pattern alone is not a puzzle: nothing maps a random texture back to a hex id, so the + only way to answer was the ink the renderer painted on the answer. The id is now legible, + which makes the task look, match, and answer. + """ + seed = int(hashlib.sha256(item_id.encode("utf-8")).hexdigest()[:16], 16) + rng = random.Random(seed) + image = Image.new("RGB", (tile_size, tile_size), color=_noise_color(rng)) + draw = ImageDraw.Draw(image) + for _ in range(96): + x0 = rng.randrange(tile_size) + y0 = rng.randrange(tile_size) + x1 = min(tile_size, x0 + rng.randrange(4, tile_size // 3)) + y1 = min(tile_size, y0 + rng.randrange(4, tile_size // 3)) + color = _noise_color(rng) + if rng.random() < 0.5: + draw.rectangle([x0, y0, x1, y1], fill=color) + else: + draw.ellipse([x0, y0, x1, y1], fill=color) + _draw_id(draw, item_id, tile_size) + image.save(path) + + +def _noise_color(rng: random.Random) -> tuple[int, int, int]: + return (rng.randrange(256), rng.randrange(256), rng.randrange(256)) + + +def _draw_id(draw: ImageDraw.ImageDraw, item_id: str, tile_size: int) -> None: + font = ImageFont.load_default(size=max(12, tile_size // 8)) + plate = ( + int(tile_size * 0.05), + int(tile_size * 0.16), + int(tile_size * 0.95), + int(tile_size * 0.88), + ) + draw.rectangle(plate, fill=(255, 255, 255)) + lines = [item_id[start : start + ID_LINE_CHARS] for start in range(0, len(item_id), ID_LINE_CHARS)] + line_height = font.getbbox("AbgJ")[3] + block_top = (plate[1] + plate[3]) / 2 - (len(lines) * line_height) / 2 + for index, line in enumerate(lines): + box = font.getbbox(line) + left = (plate[0] + plate[2]) / 2 - (box[2] - box[0]) / 2 + draw.text((left, block_top + index * line_height), line, fill=(12, 14, 18), font=font) + + +def render_visual_hash_arena( + source_path: Path, slot_paths: dict[str, Path], output_path: Path +) -> None: + """Place the source and the candidates. Nothing marks which candidate is the answer.""" + tile = 160 + canvas = Image.new("RGB", (920, 560), color=CANVAS_BACKGROUND) + draw = ImageDraw.Draw(canvas) + source_box = (72, 200, 72 + tile, 200 + tile) + slot_boxes = _slot_boxes(520, 704, 76, 324, tile) + paste_visual_hash_tile(canvas, source_path, source_box) + draw.rectangle(source_box, outline=(255, 78, 220), width=8) + for slot, box in slot_boxes.items(): + paste_visual_hash_tile(canvas, slot_paths[slot], box) + draw.rectangle(box, outline=(180, 190, 205), width=5) + _save(canvas, output_path) + + +def render_visual_hash_two_hop_arena( + source_path: Path, + intermediate_slot_paths: dict[str, Path], + target_slot_paths: dict[str, Path], + output_path: Path, +) -> None: + """Two grids of candidates, no arrow and no ring: the chain has to be read, not seen.""" + tile = 136 + canvas = Image.new("RGB", (1180, 720), color=CANVAS_BACKGROUND) + draw = ImageDraw.Draw(canvas) + source_box = (64, 292, 64 + tile, 292 + tile) + intermediate_boxes = _slot_boxes(424, 584, 104, 480, tile) + target_boxes = _slot_boxes(864, 1024, 104, 480, tile) + paste_visual_hash_tile(canvas, source_path, source_box) + draw.rectangle(source_box, outline=(255, 78, 220), width=7) + for grid, boxes, outline in ( + (intermediate_slot_paths, intermediate_boxes, (140, 215, 255)), + (target_slot_paths, target_boxes, (255, 196, 92)), + ): + for slot, box in boxes.items(): + paste_visual_hash_tile(canvas, grid[slot], box) + draw.rectangle(box, outline=outline, width=5) + _save(canvas, output_path) + + +def _slot_boxes( + left_x: int, right_x: int, top_y: int, bottom_y: int, tile: int +) -> dict[str, tuple[int, int, int, int]]: + return { + "top-left": (left_x, top_y, left_x + tile, top_y + tile), + "top-right": (right_x, top_y, right_x + tile, top_y + tile), + "bottom-left": (left_x, bottom_y, left_x + tile, bottom_y + tile), + "bottom-right": (right_x, bottom_y, right_x + tile, bottom_y + tile), + } + + +def _save(canvas: Image.Image, output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + canvas.save(output_path) + + +def paste_visual_hash_tile(canvas: Image.Image, path: Path, box: tuple[int, int, int, int]) -> None: + tile = Image.open(path).convert("RGB").resize((box[2] - box[0], box[3] - box[1]), Image.Resampling.LANCZOS) + canvas.paste(tile, box[:2]) + + +def extract_visual_target_slot(text: str) -> SlotName | None: + stripped = text.strip() + payload: dict[str, Any] | None + try: + parsed = json.loads(stripped) + payload = parsed if isinstance(parsed, dict) else None + except json.JSONDecodeError: + payload = extract_json_object(stripped) + if isinstance(payload, dict): + for key in ("target_slot", "final_slot", "slot", "answer"): + value = payload.get(key) + if isinstance(value, str): + slot = normalize_visual_slot(value) + if slot: + return slot + return normalize_visual_slot(stripped) + + +def extract_hashhop_target(text: str) -> str | None: + """Read the id the model answered with, from JSON or from loose prose.""" + stripped = text.strip() + try: + payload = json.loads(stripped) + except json.JSONDecodeError: + payload = extract_json_object(stripped) + if isinstance(payload, dict): + for key in ("answer", "target", "target_id", "final_target_id"): + value = payload.get(key) + if isinstance(value, str): + candidate = normalize_hash_id(value) + if candidate: + return candidate + match = re.search(r"\b[a-fA-F0-9]{32}\b", stripped) + return match.group(0).lower() if match else None + + +def normalize_hash_id(value: str) -> str | None: + match = re.search(r"\b[a-fA-F0-9]{32}\b", value) + return match.group(0).lower() if match else None + + +def extract_visual_intermediate_slot(text: str) -> SlotName | None: + stripped = text.strip() + try: + parsed = json.loads(stripped) + payload = parsed if isinstance(parsed, dict) else None + except json.JSONDecodeError: + payload = extract_json_object(stripped) + if isinstance(payload, dict): + for key in ("intermediate_slot", "middle_slot", "first_hop_slot"): + value = payload.get(key) + if isinstance(value, str): + slot = normalize_visual_slot(value) + if slot: + return slot + return None + + +def extract_json_object(text: str) -> dict[str, Any] | None: + start = text.find("{") + end = text.rfind("}") + if start < 0 or end <= start: + return None + try: + payload = json.loads(text[start : end + 1]) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def normalize_visual_slot(value: str) -> SlotName | None: + normalized = re.sub(r"[^a-z]+", "-", value.lower()).strip("-") + aliases: dict[str, SlotName] = { + "top-left": "top-left", + "upper-left": "top-left", + "left-top": "top-left", + "top-right": "top-right", + "upper-right": "top-right", + "right-top": "top-right", + "bottom-left": "bottom-left", + "lower-left": "bottom-left", + "left-bottom": "bottom-left", + "bottom-right": "bottom-right", + "lower-right": "bottom-right", + "right-bottom": "bottom-right", + } + if normalized in aliases: + return aliases[normalized] + for alias, slot in aliases.items(): + if re.search(rf"\b{re.escape(alias)}\b", normalized): + return slot + return None diff --git a/backend/src/ely_eye/proofs/checks_cartridge.py b/backend/src/ely_eye/proofs/checks_cartridge.py new file mode 100644 index 0000000..c814a09 --- /dev/null +++ b/backend/src/ely_eye/proofs/checks_cartridge.py @@ -0,0 +1,404 @@ +"""Cartridge assets, the memory capsule, replay, DNA, cache trace, and repository coverage. + +Moved out of suite.py so the suite order stays readable; each method is a check the +suite runs, and the suite class mixes these in. +""" + +from __future__ import annotations + +import json +import secrets +import shutil +from pathlib import Path +from typing import Any + +from ..cache_fabric import CacheFabric +from ..cartridge import CartridgeService +from ..cartridge_assets import CartridgeAssetService, merkle_root +from ..compiler import ContextCompiler +from ..config import Settings +from ..db import Database +from ..evaluation_schemas import ( + ProofCheck, +) +from ..ingestion import SUPPORTED_CODE_EXTENSIONS, build_repository_context, find_git_root +from ..schemas import ( + CartridgeManifest, + RuntimeProfile, +) + +# A fixed puzzle count per hop depth. The old suite re-rolled until the running rate crossed its +from .checks_common import ( + _semantic_counters, +) +from .report import ( + compiled_context_digest, + parquet_row_count, + read_memory_capsule, + validated_segment_hash, + write_json, +) +from .repository_checks import discover_repository_files, project_root + + +class CartridgeChecks: + """Cartridge assets, the memory capsule, replay, DNA, cache trace, and repository coverage.""" + + settings: Settings + db: Database + cartridges: CartridgeService + + def _cartridge_assets_check( + self, + cartridge_id: str, + cartridge_root: Path, + proof_root: Path, + ) -> tuple[ProofCheck, CartridgeManifest]: + report = CartridgeAssetService(self.settings, self.db).finalize(cartridge_id) + source_report = cartridge_root / "cartridge_assets.json" + proof_report = proof_root / "cartridge_assets.json" + shutil.copy2(source_report, proof_report) + rows = self.db.list_cartridges() + manifest_json = next(row["manifest_json"] for row in rows if row["cartridge_id"] == cartridge_id) + manifest = CartridgeManifest.model_validate_json(manifest_json) + expected = { + "text_vectors", + "visual_vectors", + "kv_snapshots", + "memory_capsule", + "memory_capsule_index", + "ttt_vl_adapter", + "visual_mtp_head", + "cartridge_assets", + } + by_name = {asset.name: asset for asset in report.assets} + visual_vectors = by_name.get("visual_vectors") + memory_capsule = by_name.get("memory_capsule") + gaps: list[str] = [] + if report.status != "ready": + gaps.append(f"the asset report says {report.status}") + absent = sorted(expected - set(manifest.artifacts)) + if absent: + gaps.append("not registered in the manifest: " + ", ".join(absent)) + empty = [asset.name for asset in report.assets if asset.size_bytes <= 0] + if empty: + gaps.append("zero-byte assets: " + ", ".join(empty)) + if visual_vectors is None or visual_vectors.item_count <= 0: + gaps.append("visual_vectors holds no vectors") + if memory_capsule is None or memory_capsule.item_count <= 0: + gaps.append("memory_capsule holds no segments") + valid = not gaps + return ( + ProofCheck( + name="Cartridge Physical Assets", + requirement="PRD 8.3 requires text vectors, non-empty visual vectors, KV snapshots, a 100M memory capsule, TTT-VL adapter weights, Visual-MTP head weights, and eval proofs inside the Context Cartridge.", + status="passed" if valid else "failed", + evidence=[str(proof_report), str(cartridge_root)], + detail="materialized assets: " + ", ".join(asset.name for asset in report.assets), + reason="; ".join(gaps), + rerun="ely-eye finalize-cartridge , then ely-eye proof-suite --cartridge-id ", + ), + manifest, + ) + + def _memory_capsule_check(self, cartridge_root: Path, proof_root: Path) -> ProofCheck: + capsule_path = cartridge_root / "memory_capsule.json.zst" + index_path = cartridge_root / "memory_capsule_index.json" + artifact = proof_root / "memory_capsule.json" + payload: dict[str, Any] = { + "capsule_path": str(capsule_path), + "index_path": str(index_path), + "status": "failed", + } + valid = False + try: + capsule = read_memory_capsule(capsule_path) + index = json.loads(index_path.read_text(encoding="utf-8")) if index_path.exists() else {} + raw_segments = capsule.get("segments") + if not isinstance(raw_segments, list): + raise ValueError(f"{capsule_path}: the memory capsule carries no segments list") # noqa: TRY004 # malformed stored data, not a caller type error + segments: list[dict[str, Any]] = [segment for segment in raw_segments if isinstance(segment, dict)] + if len(segments) != len(raw_segments): + raise ValueError(f"{capsule_path}: a memory capsule segment is not an object") + segment_hashes = [validated_segment_hash(segment) for segment in segments] + token_total = sum(int(segment.get("token_count") or 0) for segment in segments) + computed_merkle = merkle_root(segment_hashes) + valid = ( + capsule.get("format") == "ely-eye-memory-capsule-v1" + and int(capsule.get("token_equivalent") or 0) >= self.settings.library_target_tokens + and int(capsule.get("segment_count") or 0) == len(segments) + and token_total == int(capsule.get("token_equivalent") or -1) + and computed_merkle == capsule.get("segment_merkle_root") + and index.get("segment_merkle_root") == capsule.get("segment_merkle_root") + and int(index.get("token_equivalent") or 0) == int(capsule.get("token_equivalent") or -1) + and int(capsule.get("source_atom_count") or 0) > 0 + and all(segment_hashes) + ) + payload.update( + { + "status": "passed" if valid else "failed", + "token_equivalent": capsule.get("token_equivalent"), + "segment_count": capsule.get("segment_count"), + "source_atom_count": capsule.get("source_atom_count"), + "source_token_equivalent": capsule.get("source_token_equivalent"), + "segment_merkle_root": capsule.get("segment_merkle_root"), + "computed_merkle_root": computed_merkle, + "first_segment": segments[0] if segments else None, + "last_segment": segments[-1] if segments else None, + } + ) + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run + payload["error"] = str(exc) + write_json(artifact, payload) + return ProofCheck( + name="100M Memory Capsule", + requirement="PRD 2.2, 7.3, 8.3, and 14.3 require a 10M-100M token-equivalent Context Cartridge memory capsule with verifiable retrieval commitments.", + status="passed" if valid else "failed", + evidence=[str(artifact), str(capsule_path), str(index_path)], + detail=( + f"{payload.get('token_equivalent')} token-equivalent memory capsule with " + f"{payload.get('segment_count')} committed segments" + ), + ) + + def _cartridge_replay_check( + self, + cartridge_id: str, + cartridge_root: Path, + manifest: CartridgeManifest, + proof_root: Path, + ) -> ProofCheck: + atoms_path = cartridge_root / manifest.artifacts["atoms"] + sources_path = cartridge_root / manifest.artifacts["sources"] + sparse_path = cartridge_root / manifest.artifacts["sparse_index"] + graph_path = cartridge_root / manifest.artifacts["temporal_graph"] + computed_dna = self.cartridges.compute_dna(cartridge_root) + atom_rows = parquet_row_count(atoms_path) + try: + sources = json.loads(sources_path.read_text(encoding="utf-8")) if sources_path.exists() else [] + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{sources_path}: the cartridge source list is unreadable ({error})") from error + question = "Which Ely-Eye PRD requirements prove long context memory, HashHop, and adapters?" + compiler = ContextCompiler(self.settings, self.db) + artifact = proof_root / "cartridge_replay.json" + # The replay must come out of the cartridge, not out of whatever the live store happens + # to hold: mounting is what makes the cartridge the retrieval scope (retrieval scopes by + # the mounted set), so an unmounted replay would prove only that compiling is stable. + try: + self.cartridges.mount(cartridge_id) + except ValueError as error: + # The KV fingerprint gate refuses to mount a cartridge whose model snapshot this + # machine cannot read. A missing input is not a failed claim, so the check reports + # the refusal instead of crashing the suite. + write_json(artifact, {"verdict": "deferred", "why": str(error)}) + return ProofCheck( + name="Context Cartridge Replay", + requirement="PRD 14.2 requires the same cartridge to load reproducibly and produce consistent replay evidence.", + status="deferred", + evidence=[str(artifact), str(cartridge_root / "manifest.json")], + detail=str(error), + reason="the cartridge records a model snapshot this machine cannot verify, so the replay cannot be attempted", + rerun="set ELY_EYE_MODEL_DIR to the snapshot the cartridge was built against, then ely-eye proof-suite --cartridge-id ", + ) + try: + replay_a = compiler.compile(question, RuntimeProfile.library_100m) + replay_b = compiler.compile(question, RuntimeProfile.library_100m) + finally: + self.cartridges.unmount(cartridge_id) + digest_a = compiled_context_digest(replay_a) + digest_b = compiled_context_digest(replay_b) + payload = { + "cartridge_id": cartridge_id, + "manifest_atom_count": manifest.atom_count, + "parquet_atom_count": atom_rows, + "manifest_source_count": manifest.source_count, + "source_count": len(sources), + "dna_manifest": manifest.dna, + "dna_computed": computed_dna, + "artifacts": { + "atoms": str(atoms_path), + "sources": str(sources_path), + "sparse_index": str(sparse_path), + "temporal_graph": str(graph_path), + }, + "replay": { + "question": question, + "digest_a": digest_a, + "digest_b": digest_b, + "hit_ids_a": [hit.atom.atom_id for hit in replay_a.hits], + "hit_ids_b": [hit.atom.atom_id for hit in replay_b.hits], + "cache_trace_a": replay_a.cache_trace_id, + "cache_trace_b": replay_b.cache_trace_id, + }, + } + artifact = proof_root / "cartridge_replay.json" + write_json(artifact, payload) + cartridge_sources = {str(entry.get("source_id", "")) for entry in sources} + answered_from_cartridge = bool(replay_a.hits) and all( + hit.atom.source_id in cartridge_sources for hit in replay_a.hits + ) + valid = ( + atoms_path.exists() + and sources_path.exists() + and sparse_path.exists() + and graph_path.exists() + and atom_rows == manifest.atom_count + and len(sources) == manifest.source_count + and computed_dna == manifest.dna + and answered_from_cartridge + and digest_a == digest_b + ) + return ProofCheck( + name="Context Cartridge Replay", + requirement="PRD 14.2 requires the same cartridge to load reproducibly and produce consistent replay evidence.", + status="passed" if valid else "failed", + evidence=[str(artifact), str(cartridge_root / "manifest.json")], + detail=( + f"{atom_rows} atoms, {len(sources)} sources, replay digest {digest_a}, " + f"{len(replay_a.hits)} hits answered from the mounted cartridge" + ), + rerun="ely-eye proof-suite --cartridge-id ", + ) + + def _repository_input_check(self, proof_root: Path) -> ProofCheck: + root = project_root() + files = discover_repository_files(root) + code_files = [path for path in files if path.suffix.lower() in SUPPORTED_CODE_EXTENSIONS] + context = build_repository_context(root, files, code_files, find_git_root(root)) + artifact = proof_root / "repository_input.json" + write_json(artifact, context) + call_graph = context["code"]["call_graph"] + git_context = context["git"] + issue_pr_history = context["issue_pr_history"] + # Building a context object from files on disk is not evidence the repository reached the + # store: the claim is that the repository is queryable, so the store has to hold it. + stored_sources = [source for source in self.db.list_sources() if source.path.startswith(str(root))] + stored_atoms = self.db.atom_count() + artifact = proof_root / "repository_input.json" + payload = dict(context) + payload["store_evidence"] = { + "sources_under_project_root": len(stored_sources), + "source_ids": [source.source_id for source in stored_sources[:20]], + "atoms_in_store": stored_atoms, + } + write_json(artifact, payload) + valid = ( + context["structure"]["indexed_file_count"] > 0 + and context["code"]["indexed_file_count"] > 0 + and bool(call_graph) + and bool(git_context.get("available")) + and bool(git_context.get("recent_commits")) + and issue_pr_history.get("source") == "local_git_commit_history" + and bool(stored_sources) + and stored_atoms > 0 + ) + return ProofCheck( + name="Repository Input Coverage", + requirement="PRD 10.1 requires code repository structure indexing, call graph extraction, and issue/PR history association.", + status="passed" if valid else "failed", + evidence=[str(artifact)], + detail=( + f"{context['structure']['indexed_file_count']} files indexed; " + f"{context['code']['indexed_file_count']} code files; " + f"{len(call_graph)} call edges; " + f"{len(git_context.get('recent_commits') or [])} git commits; " + f"{len(stored_sources)} sources and {stored_atoms} atoms in the store" + ), + reason="" if valid else "the repository context was built but the store holds no atoms for it", + rerun="ely-eye ingest . && ely-eye proof-suite --cartridge-id ", + ) + + def _memory_dna_check( + self, + cartridge_root: Path, + manifest: CartridgeManifest, + proof_root: Path, + ) -> ProofCheck: + computed = self.cartridges.compute_dna(cartridge_root) + payload = { + "manifest_dna": manifest.dna, + "computed_dna": computed, + "formula": "sha256(sorted cartridge artifacts excluding manifest.json)", + "artifact_count": sum(1 for path in cartridge_root.rglob("*") if path.is_file()), + } + artifact = proof_root / "memory_dna.json" + write_json(artifact, payload) + return ProofCheck( + name="Memory DNA", + requirement="PRD 11.14 requires a reproducible Memory DNA over adapter delta, evidence graph, retrieval index, and eval proofs.", + status="passed" if manifest.dna == computed else "failed", + evidence=[str(artifact), str(cartridge_root)], + detail=f"manifest DNA {manifest.dna}", + ) + + def _cache_trace_check(self, proof_root: Path) -> ProofCheck: + """Ask the cache the same question twice and require the second answer to be a hit. + + The old check passed when any layer had any row, which a single `forget` could satisfy. + The claim in PRD 14.2 is that hit, miss, write and replay state are visible, so the check + compiles a question nobody has asked, compiles it again, and requires the semantic layer's + counters to move by exactly one miss then one hit. + """ + compiler = ContextCompiler(self.settings, self.db) + fabric = CacheFabric(self.settings, self.db) + probe = self.db.list_atoms(limit=1) + if not probe: + artifact = proof_root / "cache_trace.json" + write_json(artifact, {"verdict": "failed", "why": "the store holds no atoms"}) + return ProofCheck( + name="Cache Trace", + requirement="PRD 14.2 requires cache and memory proof with visible hit, miss, write, and replay state.", + status="failed", + evidence=[str(artifact)], + detail="the store holds no atoms, so there is nothing for the cache to answer", + reason="an empty store answers every question with an empty context, which would make the two compiles agree for the wrong reason", + rerun="ely-eye remember or ely-eye ingest first, then re-run this proof", + ) + # The nonce keeps the key unique; the quoted text keeps the question answerable, so a pass + # means the cache served a real context rather than two identical empty ones. + question = f"{probe[0].text[:60]} cache proof {secrets.token_hex(6)}" + before = _semantic_counters(fabric) + first = compiler.compile(question, RuntimeProfile.library_100m) + middle = _semantic_counters(fabric) + second = compiler.compile(question, RuntimeProfile.library_100m) + after = _semantic_counters(fabric) + digest_first = compiled_context_digest(first) + digest_second = compiled_context_digest(second) + payload = { + "question": question, + "counters": {"before": before, "after_first": middle, "after_second": after}, + "digest_first": digest_first, + "digest_second": digest_second, + "cache_trace_first": first.cache_trace_id, + "cache_trace_second": second.cache_trace_id, + } + artifact = proof_root / "cache_trace.json" + write_json(artifact, payload) + miss_recorded = middle["miss"] - before["miss"] == 1 and middle["hit"] - before["hit"] == 0 + hit_recorded = after["hit"] - middle["hit"] == 1 + same_answer = digest_first == digest_second and bool(first.hits) + valid = miss_recorded and hit_recorded and same_answer + return ProofCheck( + name="Cache Trace", + requirement="PRD 14.2 requires cache and memory proof with visible hit, miss, write, and replay state.", + status="passed" if valid else "failed", + evidence=[str(artifact)], + detail=( + f"semantic layer counters miss {before['miss']}→{middle['miss']}→{after['miss']}, " + f"hit {before['hit']}→{middle['hit']}→{after['hit']}, replay digest {digest_first}" + ), + reason="" if valid else "the second compile was not recorded as a cache hit", + rerun="ely-eye proof-suite --cartridge-id ", + ) + + def _resolve_cartridge(self, cartridge_id: str | None) -> dict[str, Any]: + rows = self.db.list_cartridges() + if not rows: + raise ValueError("No Context Cartridge is available.") + if cartridge_id is None: + return rows[0] + for row in rows: + if row["cartridge_id"] == cartridge_id: + return row + raise ValueError(f"Unknown cartridge: {cartridge_id}") + diff --git a/backend/src/ely_eye/proofs/checks_common.py b/backend/src/ely_eye/proofs/checks_common.py new file mode 100644 index 0000000..9da96a2 --- /dev/null +++ b/backend/src/ely_eye/proofs/checks_common.py @@ -0,0 +1,65 @@ +"""Shared verdict helpers for the proof checks. + +Kept out of suite.py so the check modules can import them without importing the suite. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..cache_fabric import SEMANTIC_LAYER, CacheFabric +from ..config import Settings +from ..evaluation_schemas import ( + ProofCheck, +) +from .report import ( + write_json, +) + +# A fixed puzzle count per hop depth. The old suite re-rolled until the running rate crossed its + +HASHHOP_ATTEMPTS = 8 + +GOLD_ADAPTERS = ( + "HME-Core-LoRA", + "HME-Vision-LoRA", + "HME-TTT-VL", + "HME-Visual-MTP", + "HME-Router", + "HME-Retrieval", +) + + +def _deferred_without_model( + settings: Settings, artifact: Path, name: str, requirement: str, rerun: str +) -> ProofCheck | None: + """A check that needs weights proves nothing on a machine that has no weights. + + Calling that a failure would blame the machine; calling it a pass would be a lie. The suite + records what is missing and the command that runs the check once the weights arrive. + """ + model_dir = settings.model_dir + if model_dir is not None and model_dir.exists(): + return None + write_json( + artifact, + {"status": "deferred", "reason": "no local model weights", "model_dir": str(model_dir)}, + ) + return ProofCheck( + name=name, + requirement=requirement, + status="deferred", + evidence=[str(artifact)], + detail=f"model_dir={model_dir}", + reason="the generator or encoder this check measures is not installed on this machine", + rerun=rerun, + ) + + +def _semantic_counters(fabric: CacheFabric) -> dict[str, int]: + """Hit and miss counts for the one cache layer that exists; zero when the layer has no rows.""" + for layer in fabric.statuses(): + if layer.layer == SEMANTIC_LAYER: + return {"hit": layer.hit_count, "miss": layer.miss_count} + return {"hit": 0, "miss": 0} + diff --git a/backend/src/ely_eye/proofs/checks_runtime.py b/backend/src/ely_eye/proofs/checks_runtime.py new file mode 100644 index 0000000..82f2604 --- /dev/null +++ b/backend/src/ely_eye/proofs/checks_runtime.py @@ -0,0 +1,270 @@ +"""Runtime adapter binding, gold-adapter generation, launch profiles, and the removed module check. + +Moved out of suite.py so the suite order stays readable; each method is a check the +suite runs, and the suite class mixes these in. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..compiler import ContextCompiler +from ..config import Settings +from ..db import Database +from ..evaluation_schemas import ( + ProofCheck, +) +from ..evidence_graph import EvidenceGraphService +from ..profiles import runtime_profile_report +from ..runtime import QwenRuntime, resolve_runtime_adapter +from ..schemas import ( + RuntimeProfile, + TrainingAdapterKind, +) +from ..verifier import Verifier + +# A fixed puzzle count per hop depth. The old suite re-rolled until the running rate crossed its +from .checks_common import ( + GOLD_ADAPTERS, + _deferred_without_model, +) +from .report import ( + compact_generation_context, + write_json, +) +from .training_trace import ( + count_safetensor_keys, + peft_config_loads, +) + + +class RuntimeChecks: + """Runtime adapter binding, gold-adapter generation, launch profiles, and the removed module check.""" + + settings: Settings + db: Database + + def _runtime_adapter_check(self, proof_root: Path) -> ProofCheck: + artifact = proof_root / "runtime_adapter.json" + deferred = _deferred_without_model( + self.settings, + artifact, + "Runtime Adapter Binding", + "PRD 18 requires the local model package to run with the trained adapter, and PRD 12.1 requires HME-Core-LoRA to be usable by the Qwen runtime.", + "ely-eye runtime-generation-proof with ELY_EYE_MODEL_DIR set", + ) + if deferred is not None: + return deferred + runtime = QwenRuntime(self.settings) + status = runtime.status() + adapter = resolve_runtime_adapter(self.settings) + weight_path = adapter.path / "adapter_model.safetensors" if adapter else None + payload = { + "runtime_status": status.model_dump(mode="json"), + "resolved_adapter": { + "path": str(adapter.path) if adapter else None, + "kind": adapter.manifest.kind.value if adapter else None, + "adapter_id": adapter.manifest.adapter_id if adapter else None, + "sha256": adapter.manifest.sha256 if adapter else None, + "base_model": adapter.manifest.base_model if adapter else None, + "weight_path": str(weight_path) if weight_path else None, + "safetensor_keys": count_safetensor_keys(weight_path) if weight_path else 0, + "peft_config_ok": peft_config_loads(adapter.path) if adapter else False, + }, + } + artifact = proof_root / "runtime_adapter.json" + write_json(artifact, payload) + valid = ( + status.available + and adapter is not None + and adapter.manifest.kind == TrainingAdapterKind.hme_core_lora + and adapter.manifest.base_model == self.settings.model_id + and status.adapter_path == str(adapter.path) + and weight_path is not None + and count_safetensor_keys(weight_path) > 0 + and peft_config_loads(adapter.path) + ) + return ProofCheck( + name="Runtime Adapter Binding", + requirement="PRD 18 requires the local model package to run with the trained adapter, and PRD 12.1 requires HME-Core-LoRA to be usable by the Qwen runtime.", + status="passed" if valid else "failed", + evidence=[str(artifact)], + detail=( + f"{status.backend} runtime resolves {adapter.manifest.kind.value} at {adapter.path}" + if adapter + else "runtime adapter resolution is empty" + ), + ) + + def _runtime_generation_check(self, proof_root: Path) -> ProofCheck: + artifact = proof_root / "runtime_generation.json" + deferred = _deferred_without_model( + self.settings, + artifact, + "Runtime Adapter Generation", + "PRD 12.1 and PRD 18 require the trained HME-Core-LoRA adapter to be loaded by the local Qwen runtime and used for an evidence-cited generation.", + "ely-eye proof-suite --cartridge-id with ELY_EYE_MODEL_DIR set", + ) + if deferred is not None: + return deferred + retrieval_question = ( + "Using the Ely-Eye PRD evidence, state the PRD 12.1 local training adapter " + "requirements for HME-Core-LoRA, HME-Vision-LoRA, HME-TTT-VL, " + "HME-Visual-MTP, HME-Router, and HME-Retrieval. Return JSON with atom id " + "citations." + ) + proof_settings = self.settings.model_copy(update={"max_new_tokens": 256, "temperature": 0.0}) + runtime = QwenRuntime(proof_settings) + payload: dict[str, Any] = { + "question": retrieval_question, + "status": "failed", + } + try: + compiler = ContextCompiler(proof_settings, self.db) + context = compiler.compile(retrieval_question, RuntimeProfile.library_100m) + context = compact_generation_context(context, "12.1") + if not context.hits: + raise RuntimeError("No PRD text evidence was available for runtime generation proof.") + generation_question = ( + "According to the retrieved PRD evidence, which local training adapters does " + "PRD 12.1 require? Answer with one compact JSON object: a \"citations\" field " + "naming the atom ids you actually used, and a \"sentence\" field listing them." + ) + payload["generation_question"] = generation_question + status = runtime.status() + adapter = resolve_runtime_adapter(proof_settings) + generation = runtime.generate(generation_question, context) + answer, citations, citation_report = Verifier().verify_answer(generation.text, context) + claim_id = EvidenceGraphService(proof_settings, self.db).record_answer( + generation_question, + answer, + citation_report, + context, + ) + payload.update( + { + "status": "passed", + "runtime_status": status.model_dump(mode="json"), + "adapter": { + "path": generation.adapter_path, + "kind": generation.adapter_kind, + "adapter_id": generation.adapter_id, + "sha256": generation.adapter_sha256, + }, + "tokens": { + "input": generation.input_tokens, + "output": generation.output_tokens, + }, + "context": { + "profile": context.plan.profile.value, + "hit_ids": [hit.atom.atom_id for hit in context.hits], + "token_equivalent": context.token_equivalent, + "cache_trace_id": context.cache_trace_id, + }, + "raw_generation": generation.text, + "verified_answer": answer, + "citations": citations, + "evidence_graph_claim_id": claim_id, + "verifier": citation_report.model_dump(mode="json"), + } + ) + named = [name for name in GOLD_ADAPTERS if name in answer] + cited_atoms = [ + hit.atom for hit in context.hits if hit.atom.atom_id in set(citations) + ] + supported = [atom for atom in cited_atoms if sum(name in (atom.text or "") for name in GOLD_ADAPTERS) >= 3] + valid = ( + status.available + and generation.backend == "transformers" + and adapter is not None + and generation.adapter_path == str(adapter.path) + and generation.adapter_kind == TrainingAdapterKind.hme_core_lora.value + and generation.adapter_sha256 == adapter.manifest.sha256 + and bool(context.hits) + and bool(generation.text.strip()) + and (generation.output_tokens or 0) > 0 + and len(named) >= 5 + and bool(citations) + and citation_report.citation_accuracy == 1.0 + and bool(supported) + ) + payload["answer_adapters"] = named + payload["cited_atoms"] = [atom.atom_id for atom in cited_atoms] + payload["supporting_citations"] = [atom.atom_id for atom in supported] + if not valid: + payload["status"] = "failed" + write_json(artifact, payload) + return ProofCheck( + name="Runtime Adapter Generation", + requirement="PRD 12.1 and PRD 18 require the trained HME-Core-LoRA adapter to be loaded by the local Qwen runtime and used for an evidence-cited generation.", + status="passed" if valid else "failed", + evidence=[str(artifact)], + detail=( + f"{generation.backend} generated {generation.output_tokens} tokens with " + f"{generation.adapter_kind} citation accuracy " + f"{citation_report.citation_accuracy:.2f}" + ), + ) + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run + payload["error"] = str(exc) + write_json(artifact, payload) + return ProofCheck( + name="Runtime Adapter Generation", + requirement="PRD 12.1 and PRD 18 require the trained HME-Core-LoRA adapter to be loaded by the local Qwen runtime and used for an evidence-cited generation.", + status="failed", + evidence=[str(artifact)], + detail=f"runtime generation failed: {exc}", + ) + finally: + runtime.unload() + + def _runtime_profile_check(self, proof_root: Path) -> ProofCheck: + report = runtime_profile_report() + artifact = proof_root / "runtime_profiles.json" + artifact.write_text(report.model_dump_json(indent=2), encoding="utf-8") + required_runtimes = {"SGLang", "vLLM", "KTransformers", "llama.cpp"} + runtimes = {profile.runtime for profile in report.profiles} + gaps = [f"{item.name}: {item.detail}" for item in report.validations if item.status == "missing"] + unregistered = sorted(required_runtimes - runtimes) + if unregistered: + gaps.append("no profile registered for: " + ", ".join(unregistered)) + if not gaps and report.status != "ready": + gaps.append(f"the profile report says {report.status}") + valid = not gaps + return ProofCheck( + name="Runtime Profile Delivery", + requirement="PRD 13.1 requires SGLang as the main runtime plus vLLM, KTransformers, and llama.cpp deployment profiles.", + status="passed" if valid else "failed", + evidence=[str(artifact)], + detail="registered runtimes: " + ", ".join(profile.name for profile in report.profiles), + reason="; ".join(gaps), + rerun="install the launcher in the environment the profile probes (the probe command is in runtime_profiles.json), then ely-eye runtime-profiles", + ) + + def _research_module_check(self, proof_root: Path) -> ProofCheck: + """The old version of this check could not fail, so it proved nothing. + + It compiled a query, asked the research-module service for eight module reports, and + passed when every list was non-empty. The module vectors it ranks with are SHA256-derived + placeholders, so a non-empty list is a property of the loop, not of the system. + """ + artifact = proof_root / "research_modules.json" + write_json( + artifact, + { + "claim": "PRD 11 research modules", + "verdict": "removed", + "why": "the check asserted non-empty lists built from SHA256-derived placeholder vectors", + }, + ) + return ProofCheck( + name="Research Module Delivery", + requirement="PRD 11 requires V-NSA, DVKV, X-Modal KV Sharing, Speculative Context Prefetch, Predictive KV Eviction, Learned Evidence Graph, Differentiable Retrieval Adapter, and Sleep-Time Visual Consolidation.", + status="removed", + evidence=[str(artifact)], + detail="no measurable claim exists per module; the previous check passed on non-empty lists", + reason="the check could not fail: eight named modules produced non-empty lists by construction", + rerun="write one measurable claim per module, then re-add the check", + ) + diff --git a/backend/src/ely_eye/proofs/checks_training.py b/backend/src/ely_eye/proofs/checks_training.py new file mode 100644 index 0000000..8bc9fb6 --- /dev/null +++ b/backend/src/ely_eye/proofs/checks_training.py @@ -0,0 +1,232 @@ +"""Adapter training and the learned-memory adapters bound into a cartridge. + +Moved out of suite.py so the suite order stays readable; each method is a check the +suite runs, and the suite class mixes these in. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..config import Settings +from ..db import Database +from ..evaluation_schemas import ( + ProofCheck, +) +from ..schemas import ( + AdapterManifest, + TrainingAdapterKind, +) +from ..training import ( + safetensor_weight_stats, + sha256_file, + sha256_tree, +) + +# A fixed puzzle count per hop depth. The old suite re-rolled until the running rate crossed its +from .report import ( + read_json_or_empty, + write_json, +) +from .training_trace import ( + count_safetensor_keys, + peft_config_loads, + validate_training_contract, + validate_training_proof, + validate_training_trace, +) + + +class TrainingChecks: + """Adapter training and the learned-memory adapters bound into a cartridge.""" + + settings: Settings + db: Database + required_adapters: tuple[TrainingAdapterKind, ...] + + def _adapter_training_check(self, cartridge_id: str, cartridge_root: Path, proof_root: Path) -> ProofCheck: + rows: list[dict[str, Any]] = [] + missing: list[str] = [] + manifests = self._adapter_manifests(cartridge_id) + for kind in self.required_adapters: + item = manifests.get(kind) + if item is None: + missing.append(f"{kind.value}: no adapter manifest") + continue + manifest, adapter_root = item + weight_path = adapter_root / "adapter_model.safetensors" + dataset_path = Path(manifest.dataset_path) + cartridge_adapter = cartridge_root / "adapters" / kind.value / "adapter_manifest.json" + weight_report = safetensor_weight_stats(weight_path) + keys = int(weight_report["tensor_count"]) + summary = read_json_or_empty(adapter_root / "training_summary.json") + summary_weight_sha = summary.get("adapter_weight_sha256") + summary_weight_sha_ok = ( + isinstance(summary_weight_sha, str) and summary_weight_sha == weight_report["sha256"] + ) + manifest_weight_sha_ok = ( + isinstance(manifest.adapter_weight_sha256, str) + and manifest.adapter_weight_sha256 == weight_report["sha256"] + ) + manifest_weight_stats_ok = ( + manifest.adapter_tensor_count == weight_report["tensor_count"] + and manifest.adapter_lora_tensor_count == weight_report["lora_tensor_count"] + and manifest.adapter_total_elements == weight_report["total_elements"] + and manifest.adapter_nonzero_elements == weight_report["nonzero_elements"] + and manifest.adapter_nonzero_elements > 0 + and manifest.adapter_lora_tensor_count > 0 + and manifest.adapter_weights_finite + ) + peft_ok = peft_config_loads(adapter_root) + dataset_sha_ok = dataset_path.exists() and sha256_file(dataset_path) == manifest.dataset_sha256 + tree_sha_ok = sha256_tree(adapter_root) == manifest.sha256 + cartridge_bound = cartridge_adapter.exists() + trace_report = validate_training_trace(manifest, adapter_root) + proof_report = validate_training_proof(manifest, adapter_root) + contract_report = validate_training_contract(manifest) + # Named conditions, so a failure can say which promise broke instead of listing the + # adapters that happened to be present. + conditions: list[tuple[str, bool]] = [ + ("sample_count", manifest.sample_count > 0), + ("max_steps", manifest.max_steps > 0), + ("trainable_params", manifest.trainable_params > 0), + ("final_loss recorded", manifest.final_loss is not None), + ("safetensors readable", weight_report["valid"]), + ("weight sha256 matches the training summary", summary_weight_sha_ok), + ("weight sha256 matches the manifest", manifest_weight_sha_ok), + ("weight statistics match the manifest", manifest_weight_stats_ok), + ("peft config loads", peft_ok), + ("dataset sha256 matches", dataset_sha_ok), + ("adapter tree sha256 matches", tree_sha_ok), + ("adapter is bound into the cartridge", cartridge_bound), + ("training trace verifies", trace_report["valid"]), + ("training proof verifies", proof_report["valid"]), + ("training contract verifies", contract_report["valid"]), + ] + valid = all(ok for _, ok in conditions) + if not valid: + gaps = ", ".join(name for name, ok in conditions if not ok) + missing.append(f"{kind.value}: {gaps}") + rows.append( + { + "kind": kind.value, + "adapter": str(adapter_root), + "dataset": str(dataset_path), + "sample_count": manifest.sample_count, + "max_steps": manifest.max_steps, + "trainable_params": manifest.trainable_params, + "final_loss": manifest.final_loss, + "safetensor_keys": keys, + "adapter_weight_sha256": weight_report["sha256"], + "adapter_weight_sha256_ok": summary_weight_sha_ok and manifest_weight_sha_ok, + "adapter_weight_stats_ok": manifest_weight_stats_ok, + "adapter_weight_stats": weight_report, + "dataset_sha256_ok": dataset_sha_ok, + "adapter_sha256_ok": tree_sha_ok, + "peft_config_ok": peft_ok, + "cartridge_bound": cartridge_bound, + "training_contract": contract_report, + "training_trace": trace_report, + "training_proof": proof_report, + } + ) + artifact = proof_root / "adapter_training.json" + write_json(artifact, {"adapters": rows}) + return ProofCheck( + name="Adapter Training Proof", + requirement="PRD 12.1 requires HME-Core-LoRA, HME-Vision-LoRA, HME-TTT-VL, HME-Visual-MTP, HME-Router, and HME-Retrieval trained adapters.", + status="failed" if missing else "passed", + evidence=[str(artifact), str(self.settings.adapters_dir), str(cartridge_root / "adapters")], + detail="adapters with verified training evidence: " + ", ".join(row["kind"] for row in rows), + reason="; ".join(missing), + rerun="train the missing adapters (ely-eye train-lora) or attach them (ely-eye attach-adapter), then ely-eye proof-suite --cartridge-id ", + ) + + def _learned_memory_recall_check( + self, + cartridge_id: str, + cartridge_root: Path, + proof_root: Path, + ) -> ProofCheck: + manifests = self._adapter_manifests(cartridge_id) + learned_kinds = ( + TrainingAdapterKind.hme_ttt_vl, + TrainingAdapterKind.hme_core_lora, + TrainingAdapterKind.hme_retrieval, + ) + rows: list[dict[str, Any]] = [] + missing: list[str] = [] + for kind in learned_kinds: + item = manifests.get(kind) + if item is None: + missing.append(f"{kind.value}: no adapter manifest") + continue + manifest, adapter_root = item + weight_path = adapter_root / "adapter_model.safetensors" + cartridge_weight = cartridge_root / "adapters" / kind.value / "adapter_model.safetensors" + rows.append( + { + "kind": kind.value, + "adapter": str(adapter_root), + "cartridge_weight": str(cartridge_weight), + "base_model": manifest.base_model, + "dataset_sha256": manifest.dataset_sha256, + "sample_count": manifest.sample_count, + "trainable_params": manifest.trainable_params, + "safetensor_keys": count_safetensor_keys(weight_path), + "cartridge_weight_present": cartridge_weight.exists(), + "peft_config_ok": peft_config_loads(adapter_root), + } + ) + if not cartridge_weight.exists(): + missing.append(f"{kind.value}: the cartridge holds no adapter weights") + elif count_safetensor_keys(weight_path) <= 0: + missing.append(f"{kind.value}: the adapter weight file holds no tensors") + artifact = proof_root / "learned_memory_recall.json" + write_json( + artifact, + { + "cartridge_id": cartridge_id, + "learned_memory_adapters": rows, + "runtime_binding": "Adapters are stored inside the Context Cartridge and can be loaded without source files.", + }, + ) + return ProofCheck( + name="Learned Memory Recall", + requirement="PRD 14.2 and 15.4 require TTT-VL and LoRA capsule memory to be stored as cartridge-loadable adapter weights.", + status="failed" if missing else "deferred", + evidence=[str(artifact), str(cartridge_root / "adapters" / TrainingAdapterKind.hme_ttt_vl.value)], + detail=( + "weights load for " + ", ".join(row["kind"] for row in rows) + + "; no recall number was measured" + ), + reason=( + "the check loads adapter weights and never asks the model to remember anything, so it " + "cannot carry the word recall; the load evidence is real, the recall claim is not yet measured" + ), + rerun="add a recall probe that answers questions with and without the adapter, then ely-eye proof-suite --cartridge-id ", + ) + + def _adapter_manifests( + self, + cartridge_id: str, + ) -> dict[TrainingAdapterKind, tuple[AdapterManifest, Path]]: + candidates: dict[TrainingAdapterKind, list[tuple[AdapterManifest, Path]]] = {} + for path in sorted(self.settings.adapters_dir.glob("*/adapter_manifest.json")): + manifest = AdapterManifest.model_validate_json(path.read_text(encoding="utf-8")) + candidates.setdefault(manifest.kind, []).append((manifest, path.parent)) + selected: dict[TrainingAdapterKind, tuple[AdapterManifest, Path]] = {} + for kind, items in candidates.items(): + items.sort( + key=lambda item: ( + item[0].cartridge_id == cartridge_id, + (item[1] / "adapter_model.safetensors").stat().st_mtime + if (item[1] / "adapter_model.safetensors").exists() + else 0.0, + ), + reverse=True, + ) + selected[kind] = items[0] + return selected + diff --git a/backend/src/ely_eye/proofs/checks_visual.py b/backend/src/ely_eye/proofs/checks_visual.py new file mode 100644 index 0000000..622d061 --- /dev/null +++ b/backend/src/ely_eye/proofs/checks_visual.py @@ -0,0 +1,315 @@ +"""Visual HashHop retrieval, the benchmark registry, and the visual contradiction lens. + +Moved out of suite.py so the suite order stays readable; each method is a check the +suite runs, and the suite class mixes these in. +""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +from ..benchmarks import ( + benchmark_prediction_report, + benchmark_registry_report, + benchmark_sample_report, + benchmark_source_report, +) +from ..config import Settings +from ..db import Database +from ..evaluation_schemas import ( + ProofCheck, +) +from ..runtime import QwenRuntime +from ..verifier import VisualContradictionLens + +# A fixed puzzle count per hop depth. The old suite re-rolled until the running rate crossed its +from .checks_common import ( + HASHHOP_ATTEMPTS, + _deferred_without_model, +) +from .contradictions import CORPUS_NAME, corpus_path, load_corpus, scenario_context +from .hashhop import HashHopEvaluator +from .hashhop_verify import ( + collect_model_visual_proofs, + copy_proof_artifacts, + hashhop_chain_valid, + pass_rate, +) +from .report import ( + format_scores, + write_json, +) + + +class VisualChecks: + """Visual HashHop retrieval, the benchmark registry, and the visual contradiction lens.""" + + settings: Settings + db: Database + + def _visual_hashhop_check(self, proof_root: Path) -> ProofCheck: + artifact = proof_root / "visual_hashhop_suite.json" + deferred = _deferred_without_model( + self.settings, + artifact, + "Visual HashHop Retrieval", + "PRD 14.2 requires the model to resolve visual hash hops inside 262K, 1.01M and 100M token contexts.", + "ely-eye hashhop visual 1 262144 with ELY_EYE_MODEL_DIR set", + ) + if deferred is not None: + return deferred + proof_settings = self.settings.model_copy(update={"max_new_tokens": 128, "temperature": 0.0}) + runtime = QwenRuntime(proof_settings) + evaluator = HashHopEvaluator(proof_settings, runtime) + try: + base_proofs = [ + evaluator.generate_visual_proof(1, 262_144), + evaluator.generate_visual_proof(2, 262_144), + evaluator.generate_visual_proof(1, 1_010_000), + evaluator.generate_visual_proof(1, 100_000_000), + evaluator.generate_text_proof(2, 1_010_000), + evaluator.generate_model_text_proof(2, 1_010_000), + ] + model_visual_1hop = collect_model_visual_proofs(evaluator, 1, 262_144, HASHHOP_ATTEMPTS) + model_visual_2hop = collect_model_visual_proofs(evaluator, 2, 262_144, HASHHOP_ATTEMPTS) + proofs = base_proofs + model_visual_1hop + model_visual_2hop + finally: + runtime.unload() + payloads: list[dict[str, Any]] = [] + for proof in proofs: + target = proof_root / "hashhop" / proof.proof_id + copy_proof_artifacts(proof, target) + payloads.append( + { + "proof": proof.model_dump(mode="json"), + "chain_valid": hashhop_chain_valid(proof), + "artifact_dir": str(target), + } + ) + artifact = proof_root / "visual_hashhop_suite.json" + model_solved = [ + item + for item in payloads + if item["proof"]["model_target_id"] is not None or item["proof"]["passed"] is not None + ] + model_text_solved = [ + proof + for proof in base_proofs + if proof.kind == "text_hashhop" and proof.model_target_id is not None + ] + model_solved_passed = [item for item in model_solved if item["proof"]["passed"]] + visual_1hop_rate = pass_rate(model_visual_1hop) + visual_2hop_rate = pass_rate(model_visual_2hop) + metrics = { + "visual_hashhop_262k_1hop": { + "target": 0.90, + "sample_count": len(model_visual_1hop), + "passed": sum(1 for proof in model_visual_1hop if proof.passed), + "pass_rate": visual_1hop_rate, + }, + "visual_hashhop_262k_2hop": { + "target": 0.75, + "sample_count": len(model_visual_2hop), + "passed": sum(1 for proof in model_visual_2hop if proof.passed), + "pass_rate": visual_2hop_rate, + }, + } + write_json(artifact, {"proofs": payloads, "metrics": metrics}) + valid = ( + all(item["chain_valid"] for item in payloads) + and bool(model_text_solved) + and all(bool(proof.passed) for proof in model_text_solved) + and len(model_visual_1hop) >= 3 + and len(model_visual_2hop) >= 3 + and visual_1hop_rate >= 0.90 + and visual_2hop_rate >= 0.75 + ) + return ProofCheck( + name="Visual HashHop", + requirement="PRD 14.2 and 14.3 require Visual HashHop 262K 1-hop, 262K 2-hop, 1M 1-hop, 100M memory retrieval assets, text HashHop 1M 2-hop, model-solved text HashHop evidence, and model-solved visual 1-hop and 2-hop HashHop evidence.", + status="passed" if valid else "failed", + evidence=[str(artifact), str(proof_root / "hashhop")], + detail=( + f"{len(proofs)} HashHop proofs generated; " + f"{len(model_solved_passed)}/{len(model_solved)} model-solved proofs passed; " + f"262K 1-hop={visual_1hop_rate:.2f}, 2-hop={visual_2hop_rate:.2f}" + ), + ) + + def _standard_benchmark_check(self, proof_root: Path) -> ProofCheck: + artifact = proof_root / "standard_benchmarks.json" + deferred = _deferred_without_model( + self.settings, + artifact, + "Standard Benchmark Registry", + "PRD 14.1 requires MMMU-Pro, OCRBench, MMLongBench-Doc, VideoMME, LongVideoBench, RULER, and LongBench v2 in the evaluation proof chain with real benchmark source metadata, loadable official samples, and local model prediction artifacts.", + "ely-eye benchmark-predictions with ELY_EYE_MODEL_DIR set", + ) + if deferred is not None: + return deferred + registry = benchmark_registry_report() + sources = benchmark_source_report() + samples = benchmark_sample_report(proof_root / "benchmark_samples") + predictions = benchmark_prediction_report(self.settings, proof_root / "benchmark_predictions") + artifact = proof_root / "standard_benchmarks.json" + source_artifact = proof_root / "standard_benchmark_sources.json" + sample_artifact = proof_root / "standard_benchmark_samples.json" + prediction_artifact = proof_root / "standard_benchmark_predictions.json" + artifact.write_text(registry.model_dump_json(indent=2), encoding="utf-8") + source_artifact.write_text(sources.model_dump_json(indent=2), encoding="utf-8") + sample_artifact.write_text(samples.model_dump_json(indent=2), encoding="utf-8") + prediction_artifact.write_text(predictions.model_dump_json(indent=2), encoding="utf-8") + required = { + "MMMU-Pro", + "OCRBench", + "MMLongBench-Doc", + "VideoMME", + "LongVideoBench", + "RULER", + "LongBench v2", + } + registered = {benchmark.name for benchmark in registry.benchmarks} + source_names = {source.name for source in sources.sources if source.status == "ready" and source.sha} + loaded_sample_names = {sample.name for sample in samples.samples if sample.status == "loaded"} + gated_sample_names = {sample.name for sample in samples.samples if sample.status == "gated"} + source_proved_sample_names = { + sample.name + for sample in samples.samples + if sample.status == "loaded" and sample.sample_sha256 and sample.source_dataset and sample.source_sha + } + predicted_sample_names = { + prediction.name for prediction in predictions.predictions if prediction.status == "predicted" + } + prediction_scores = { + prediction.name: prediction.score + for prediction in predictions.predictions + if prediction.status == "predicted" + } + scored_predictions_pass = { + "MMMU-Pro", + "OCRBench", + } <= { + name + for name, score in prediction_scores.items() + if isinstance(score, (int, float)) and math.isfinite(float(score)) and float(score) >= 0.5 + } + sample_names = loaded_sample_names | gated_sample_names + valid = ( + registry.status == "ready" + and sources.status == "ready" + and samples.status == "ready" + and predictions.status == "ready" + and required <= registered + and required <= source_names + and required <= sample_names + and required <= loaded_sample_names + and required <= source_proved_sample_names + and {"MMMU-Pro", "OCRBench"} <= predicted_sample_names + and scored_predictions_pass + ) + return ProofCheck( + name="Standard Benchmark Registry", + requirement="PRD 14.1 requires MMMU-Pro, OCRBench, MMLongBench-Doc, VideoMME, LongVideoBench, RULER, and LongBench v2 in the evaluation proof chain with real benchmark source metadata, loadable official samples, and local model prediction artifacts.", + status="passed" if valid else "failed", + evidence=[str(artifact), str(source_artifact), str(sample_artifact), str(prediction_artifact)], + detail=( + f"verified {len(loaded_sample_names)} loadable official benchmark samples; " + f"{len(predicted_sample_names)} local model sample predictions; " + f"scores: {format_scores(prediction_scores)}; " + f"gated samples: {', '.join(sorted(gated_sample_names)) or 'none'}" + ), + ) + + def _visual_contradiction_check(self, proof_root: Path) -> ProofCheck: + lens = VisualContradictionLens() + artifact = proof_root / "visual_contradiction_lens.json" + path = corpus_path(self.settings.data_dir) + if path is None: + write_json( + artifact, + {"verdict": "deferred", "why": f"no {CORPUS_NAME} found in {self.settings.data_dir}"}, + ) + return ProofCheck( + name="Visual Contradiction Lens", + requirement="PRD 14.2 and 14.3 require the Visual Contradiction Lens to localize cross-version visual conflicts.", + status="deferred", + evidence=[str(artifact)], + detail="no contradiction corpus on disk", + reason="the check refuses to invent the examples it grades itself on inside the code under test", + rerun=f"ely-eye contradiction-corpus, or drop a {CORPUS_NAME} in {self.settings.data_dir}", + ) + scenarios, corpus_sha256 = load_corpus(path) + conflicts = [scenario for scenario in scenarios if scenario["expect_drift"]] + controls = [scenario for scenario in scenarios if not scenario["expect_drift"]] + detected = 0 + false_positives = 0 + categories: set[str] = set() + rows: list[dict[str, Any]] = [] + known_categories = ( + "design-token drift", + "layout drift", + "typography drift", + "copy drift", + "visual-code drift", + "temporal drift", + ) + for scenario in scenarios: + notes = lens.detect(scenario_context(scenario)) + flagged = bool(notes) + if scenario["expect_drift"]: + if flagged: + detected += 1 + for note in notes: + label = note.split(":", 1)[0] + categories.update(category for category in known_categories if label.endswith(category)) + elif flagged: + false_positives += 1 + rows.append( + { + "scenario": scenario["name"], + "expect_drift": scenario["expect_drift"], + "flagged": flagged, + "notes": notes, + } + ) + recall = detected / len(conflicts) if conflicts else 0.0 + false_positive_rate = false_positives / len(controls) if controls else 0.0 + valid = recall >= 0.80 and false_positive_rate == 0.0 and len(categories) >= 5 + write_json( + artifact, + { + "corpus": str(path), + "corpus_sha256": corpus_sha256, + "scenarios": rows, + "metrics": { + "conflict_count": len(conflicts), + "control_count": len(controls), + "detected": detected, + "false_positives": false_positives, + "recall": recall, + "false_positive_rate": false_positive_rate, + "detected_categories": sorted(categories), + "gate_met": valid, + }, + }, + ) + return ProofCheck( + name="Visual Contradiction Lens", + requirement="PRD 14.2 and 14.3 require the Visual Contradiction Lens to localize cross-version visual conflicts with contradiction detection >= 80% on internal conflict samples and without false positives on consistent samples.", + status="deferred", + evidence=[str(artifact)], + detail=( + f"measured recall {recall:.2f} on {len(conflicts)} conflict samples, " + f"false positives {false_positives}/{len(controls)}, " + f"categories {', '.join(sorted(categories))}" + ), + reason=( + f"the lens is graded on {CORPUS_NAME} (sha256 {corpus_sha256[:16]}…), a corpus this repo " + "authored, and detects it with five regexes over atom text; a producer grading itself on " + "its own examples is not evidence, so the number is reported but not claimed" + ), + rerun="drop a third-party UI-drift set at $ELY_EYE_HOME/contradiction_scenarios.json (ely-eye contradiction-corpus writes the default), then ely-eye visual-contradiction-proof", + ) + diff --git a/backend/src/ely_eye/proofs/contradiction_scenarios.json b/backend/src/ely_eye/proofs/contradiction_scenarios.json new file mode 100644 index 0000000..b62cc50 --- /dev/null +++ b/backend/src/ely_eye/proofs/contradiction_scenarios.json @@ -0,0 +1,927 @@ +[ + { + "name": "button-radius-token-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "home_btn_v4", + "modality": "ui_screenshot", + "source_id": "home_v4", + "source": "home_v4.figma", + "ingested_at": "2026-10-07T22:07:05.953200Z", + "text": "button.radius: 8px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "figma" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "checkout_btn_v4", + "modality": "ui_screenshot", + "source_id": "checkout_v4", + "source": "checkout_v4.figma", + "ingested_at": "2026-10-07T22:07:05.953230Z", + "text": "button.radius: 12px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "figma" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "primary-color-token-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "spec_color", + "modality": "pdf_page", + "source_id": "brand_spec", + "source": "brand_guide.pdf", + "ingested_at": "2026-10-07T22:07:05.953237Z", + "text": "primary.color: #2563eb", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "spec" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "impl_color", + "modality": "code", + "source_id": "theme_css", + "source": "src/theme.css", + "ingested_at": "2026-10-07T22:07:05.953241Z", + "text": "primary.color: #1d4ed8", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "heading-typography-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "home_heading", + "modality": "ui_screenshot", + "source_id": "home_shot", + "source": "home_v3.png", + "ingested_at": "2026-10-07T22:07:05.953245Z", + "text": "heading.font-size: 24px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "ui_screenshot" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "checkout_heading", + "modality": "ui_screenshot", + "source_id": "checkout_shot", + "source": "checkout_v3.png", + "ingested_at": "2026-10-07T22:07:05.953248Z", + "text": "heading.font-size: 28px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "ui_screenshot" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "card-spacing-layout-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "card_home", + "modality": "ui_screenshot", + "source_id": "card_home_shot", + "source": "home_v3.png", + "ingested_at": "2026-10-07T22:07:05.953252Z", + "text": "card.padding: 16px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "ui_screenshot" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "card_settings", + "modality": "ui_screenshot", + "source_id": "card_settings_shot", + "source": "settings_v3.png", + "ingested_at": "2026-10-07T22:07:05.953255Z", + "text": "card.padding: 24px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "ui_screenshot" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "cta-copy-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "cta_v1", + "modality": "ui_screenshot", + "source_id": "shot_v1", + "source": "cart_v1.png", + "ingested_at": "2026-10-07T22:07:05.953258Z", + "text": "cta.label: \"Checkout\"", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "ui_screenshot" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "cta_v4", + "modality": "ui_screenshot", + "source_id": "shot_v4", + "source": "cart_v4.png", + "ingested_at": "2026-10-07T22:07:05.953263Z", + "text": "cta.label: \"Pay now\"", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "ui_screenshot" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "banner-radius-temporal-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "banner_v1", + "modality": "ui_screenshot", + "source_id": "banner_spec_v1", + "source": "banner_v1.figma", + "ingested_at": "2026-10-07T22:07:05.953267Z", + "text": "banner.radius: 4px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "figma", + "version": "v1" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "banner_v4", + "modality": "ui_screenshot", + "source_id": "banner_spec_v4", + "source": "banner_v4.figma", + "ingested_at": "2026-10-07T22:07:05.953271Z", + "text": "banner.radius: 10px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "figma", + "version": "v4" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "toggle-accent-visual-code-drift", + "expect_drift": true, + "atoms": [ + { + "atom_id": "toggle_design", + "modality": "ui_screenshot", + "source_id": "toggle_figma", + "source": "toggle.figma", + "ingested_at": "2026-10-07T22:07:05.953274Z", + "text": "toggle.color: #10b981", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "figma" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "toggle_code", + "modality": "code", + "source_id": "toggle_css", + "source": "src/toggle.css", + "ingested_at": "2026-10-07T22:07:05.953279Z", + "text": "toggle.color: #059669", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "consistent-button-radius", + "expect_drift": false, + "atoms": [ + { + "atom_id": "ctrl_btn_spec", + "modality": "pdf_page", + "source_id": "ctrl_spec", + "source": "spec.pdf", + "ingested_at": "2026-10-07T22:07:05.953282Z", + "text": "button.radius: 8px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "spec" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "ctrl_btn_css", + "modality": "code", + "source_id": "ctrl_css", + "source": "src/tokens.css", + "ingested_at": "2026-10-07T22:07:05.953284Z", + "text": "button.radius = 8px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "distinct-components-not-a-conflict", + "expect_drift": false, + "atoms": [ + { + "atom_id": "ctrl_button_radius", + "modality": "code", + "source_id": "ctrl_a", + "source": "a.css", + "ingested_at": "2026-10-07T22:07:05.953288Z", + "text": "button.radius: 8px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "ctrl_input_radius", + "modality": "code", + "source_id": "ctrl_b", + "source": "b.css", + "ingested_at": "2026-10-07T22:07:05.953293Z", + "text": "input.radius: 12px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 4, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "single-source-multiple-rules", + "expect_drift": false, + "atoms": [ + { + "atom_id": "ctrl_single_css", + "modality": "code", + "source_id": "ctrl_single", + "source": "src/base.css", + "ingested_at": "2026-10-07T22:07:05.953297Z", + "text": "header.padding: 8px; footer.padding: 24px; aside.padding: 16px", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 15, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "consistent-primary-color", + "expect_drift": false, + "atoms": [ + { + "atom_id": "ctrl_color_spec", + "modality": "pdf_page", + "source_id": "ctrl_color_a", + "source": "brand.pdf", + "ingested_at": "2026-10-07T22:07:05.953300Z", + "text": "primary.color: #2563eb", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "spec" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "ctrl_color_css", + "modality": "code", + "source_id": "ctrl_color_b", + "source": "theme.css", + "ingested_at": "2026-10-07T22:07:05.953302Z", + "text": "primary.color: #2563eb", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 5, + "metadata": { + "source_kind": "css" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + }, + { + "name": "prose-without-design-facts", + "expect_drift": false, + "atoms": [ + { + "atom_id": "ctrl_prose_a", + "modality": "text", + "source_id": "ctrl_prose_1", + "source": "notes_a.txt", + "ingested_at": "2026-10-07T22:07:05.953306Z", + "text": "The onboarding flow feels smooth and the colors look balanced.", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 15, + "metadata": { + "source_kind": "text" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + }, + { + "atom_id": "ctrl_prose_b", + "modality": "text", + "source_id": "ctrl_prose_2", + "source": "notes_b.txt", + "ingested_at": "2026-10-07T22:07:05.953308Z", + "text": "Users reported the checkout button is easy to find.", + "image_ref": null, + "embedding_refs": [], + "layout": null, + "relations": [], + "trust": { + "parser": 0.95, + "ocr": 0.9, + "model": 0.85 + }, + "token_equivalent": 12, + "metadata": { + "source_kind": "text" + }, + "holder_id": "", + "origin": "artifact", + "scenario": "", + "event_time": null, + "event_time_end": null, + "event_time_precision": "unknown", + "event_time_source": "", + "valid_from": null, + "valid_to": null, + "supersedes": [], + "visibility": { + "scope": "holder", + "holders": [], + "purpose": null + }, + "retention_until": null + } + ] + } +] diff --git a/backend/src/ely_eye/proofs/contradictions.py b/backend/src/ely_eye/proofs/contradictions.py new file mode 100644 index 0000000..65730ea --- /dev/null +++ b/backend/src/ely_eye/proofs/contradictions.py @@ -0,0 +1,235 @@ +"""Contradiction scenarios and the lens that has to spot them. + +The scenarios are data, not code: the proof loads them from a JSON corpus so the set that grades +the lens is reviewable, replaceable, and named by hash in the proof artifact. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from ..schemas import ( + CompiledContext, + ContextPlan, + EvidenceAtom, + Modality, + RetrievalHit, + RuntimeProfile, + TrustScores, +) + +CORPUS_NAME = "contradiction_scenarios.json" +PACKAGED_CORPUS = Path(__file__).with_name(CORPUS_NAME) + + +def contradiction_atom( + atom_id: str, + source_id: str, + source: str, + text: str, + modality: Modality, + source_kind: str, + version: str | None = None, +) -> EvidenceAtom: + metadata: dict[str, Any] = {"source_kind": source_kind} + if version is not None: + metadata["version"] = version + return EvidenceAtom( + atom_id=atom_id, + modality=modality, + source_id=source_id, + source=source, + text=text, + trust=TrustScores(parser=0.95, ocr=0.9, model=0.85), + token_equivalent=max(1, len(text) // 4), + metadata=metadata, + ) + + +def contradiction_context(question: str, atoms: list[EvidenceAtom]) -> CompiledContext: + hits = [ + RetrievalHit(atom=atom, sparse_score=0.8, dense_score=0.0, graph_score=0.0, final_score=0.8) + for atom in atoms + ] + plan = ContextPlan( + question=question, + profile=RuntimeProfile.research_theater, + required_modalities=sorted({atom.modality for atom in atoms}), + evidence_budget_atoms=len(atoms), + token_budget=sum(atom.token_equivalent for atom in atoms), + retrieval_query=question, + compression_strategy="visual-contradiction-eval", + verifier_contract=["Report contradictions as contradiction_notes."], + ) + packed = "\n".join(f"[{atom.atom_id}] {atom.text}" for atom in atoms) + return CompiledContext( + plan=plan, + hits=hits, + packed_text=packed, + token_equivalent=sum(atom.token_equivalent for atom in atoms), + cache_trace_id="visual_contradiction_eval", + ) + + +def build_contradiction_scenarios() -> list[dict[str, Any]]: + """Deterministic cross-version conflict and control samples (PRD 11.15). + + Each conflict pair injects a single genuine drift across two distinct + sources; each control pair is internally consistent or scoped to different + components. The samples exercise the live ``VisualContradictionLens`` so the + proof measures real recall and false-positive rate. + + A scenario carries its atoms and its label; the compiled context the lens reads is built at + use, so the corpus on disk holds no derived state. + """ + + scenarios: list[dict[str, Any]] = [] + + def conflict(name: str, atoms: list[EvidenceAtom]) -> None: + scenarios.append({"name": name, "atoms": atoms, "expect_drift": True}) + + def control(name: str, atoms: list[EvidenceAtom]) -> None: + scenarios.append({"name": name, "atoms": atoms, "expect_drift": False}) + + conflict( + "button-radius-token-drift", + [ + contradiction_atom("home_btn_v4", "home_v4", "home_v4.figma", "button.radius: 8px", Modality.ui_screenshot, "figma"), + contradiction_atom("checkout_btn_v4", "checkout_v4", "checkout_v4.figma", "button.radius: 12px", Modality.ui_screenshot, "figma"), + ], + ) + conflict( + "primary-color-token-drift", + [ + contradiction_atom("spec_color", "brand_spec", "brand_guide.pdf", "primary.color: #2563eb", Modality.pdf_page, "spec"), + contradiction_atom("impl_color", "theme_css", "src/theme.css", "primary.color: #1d4ed8", Modality.code, "css"), + ], + ) + conflict( + "heading-typography-drift", + [ + contradiction_atom("home_heading", "home_shot", "home_v3.png", "heading.font-size: 24px", Modality.ui_screenshot, "ui_screenshot"), + contradiction_atom("checkout_heading", "checkout_shot", "checkout_v3.png", "heading.font-size: 28px", Modality.ui_screenshot, "ui_screenshot"), + ], + ) + conflict( + "card-spacing-layout-drift", + [ + contradiction_atom("card_home", "card_home_shot", "home_v3.png", "card.padding: 16px", Modality.ui_screenshot, "ui_screenshot"), + contradiction_atom("card_settings", "card_settings_shot", "settings_v3.png", "card.padding: 24px", Modality.ui_screenshot, "ui_screenshot"), + ], + ) + conflict( + "cta-copy-drift", + [ + contradiction_atom("cta_v1", "shot_v1", "cart_v1.png", 'cta.label: "Checkout"', Modality.ui_screenshot, "ui_screenshot"), + contradiction_atom("cta_v4", "shot_v4", "cart_v4.png", 'cta.label: "Pay now"', Modality.ui_screenshot, "ui_screenshot"), + ], + ) + conflict( + "banner-radius-temporal-drift", + [ + contradiction_atom("banner_v1", "banner_spec_v1", "banner_v1.figma", "banner.radius: 4px", Modality.ui_screenshot, "figma", version="v1"), + contradiction_atom("banner_v4", "banner_spec_v4", "banner_v4.figma", "banner.radius: 10px", Modality.ui_screenshot, "figma", version="v4"), + ], + ) + conflict( + "toggle-accent-visual-code-drift", + [ + contradiction_atom("toggle_design", "toggle_figma", "toggle.figma", "toggle.color: #10b981", Modality.ui_screenshot, "figma"), + contradiction_atom("toggle_code", "toggle_css", "src/toggle.css", "toggle.color: #059669", Modality.code, "css"), + ], + ) + + control( + "consistent-button-radius", + [ + contradiction_atom("ctrl_btn_spec", "ctrl_spec", "spec.pdf", "button.radius: 8px", Modality.pdf_page, "spec"), + contradiction_atom("ctrl_btn_css", "ctrl_css", "src/tokens.css", "button.radius = 8px", Modality.code, "css"), + ], + ) + control( + "distinct-components-not-a-conflict", + [ + contradiction_atom("ctrl_button_radius", "ctrl_a", "a.css", "button.radius: 8px", Modality.code, "css"), + contradiction_atom("ctrl_input_radius", "ctrl_b", "b.css", "input.radius: 12px", Modality.code, "css"), + ], + ) + control( + "single-source-multiple-rules", + [ + contradiction_atom( + "ctrl_single_css", + "ctrl_single", + "src/base.css", + "header.padding: 8px; footer.padding: 24px; aside.padding: 16px", + Modality.code, + "css", + ), + ], + ) + control( + "consistent-primary-color", + [ + contradiction_atom("ctrl_color_spec", "ctrl_color_a", "brand.pdf", "primary.color: #2563eb", Modality.pdf_page, "spec"), + contradiction_atom("ctrl_color_css", "ctrl_color_b", "theme.css", "primary.color: #2563eb", Modality.code, "css"), + ], + ) + control( + "prose-without-design-facts", + [ + contradiction_atom("ctrl_prose_a", "ctrl_prose_1", "notes_a.txt", "The onboarding flow feels smooth and the colors look balanced.", Modality.text, "text"), + contradiction_atom("ctrl_prose_b", "ctrl_prose_2", "notes_b.txt", "Users reported the checkout button is easy to find.", Modality.text, "text"), + ], + ) + + return scenarios + + +def corpus_path(data_dir: Path) -> Path | None: + """A corpus the operator dropped in beats the packaged one; neither beats silence.""" + override = data_dir / CORPUS_NAME + if override.exists(): + return override + return PACKAGED_CORPUS if PACKAGED_CORPUS.exists() else None + + +def write_corpus(path: Path) -> Path: + """Freeze the generated scenarios as data so the grader can be read, not just run.""" + payload = [ + {"name": scenario["name"], "expect_drift": scenario["expect_drift"], "atoms": [atom.model_dump(mode="json") for atom in scenario["atoms"]]} + for scenario in build_contradiction_scenarios() + ] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return path + + +def load_corpus(path: Path) -> tuple[list[dict[str, Any]], str]: + """Read the corpus and its hash. A scenario whose label cannot be parsed fails loudly.""" + try: + raw = path.read_bytes() + rows = json.loads(raw) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{path}: the contradiction corpus is unreadable ({error})") from error + if not isinstance(rows, list) or not rows: + raise ValueError(f"{path}: the contradiction corpus must be a non-empty list") + scenarios: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict) or "name" not in row or "expect_drift" not in row: + raise ValueError(f"{path}: a contradiction scenario is missing its name or its label") + try: + atoms = [EvidenceAtom.model_validate(entry) for entry in row.get("atoms", [])] + except ValueError as error: + raise ValueError(f"{path}: scenario {row['name']!r} holds an unreadable atom ({error})") from error + if not atoms: + raise ValueError(f"{path}: scenario {row['name']!r} has no atoms to compare") + scenarios.append({"name": row["name"], "expect_drift": bool(row["expect_drift"]), "atoms": atoms}) + return scenarios, hashlib.sha256(raw).hexdigest() + + +def scenario_context(scenario: dict[str, Any]) -> CompiledContext: + return contradiction_context(str(scenario["name"]), scenario["atoms"]) diff --git a/backend/src/ely_eye/proofs/hashhop.py b/backend/src/ely_eye/proofs/hashhop.py new file mode 100644 index 0000000..b18dbd7 --- /dev/null +++ b/backend/src/ely_eye/proofs/hashhop.py @@ -0,0 +1,430 @@ +"""Build HashHop puzzles and ask a model to walk them. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import secrets +from pathlib import Path +from typing import Any + +from ..config import Settings, get_settings +from ..evaluation_schemas import ( + HashHopProof, +) +from ..runtime import QwenRuntime +from .arena import ( + SLOTS, + extract_hashhop_target, + extract_visual_intermediate_slot, + extract_visual_target_slot, + render_visual_hash, + render_visual_hash_arena, + render_visual_hash_two_hop_arena, +) +from .report import write_json + + +class HashHopEvaluator: + def __init__(self, settings: Settings | None = None, runtime: QwenRuntime | None = None) -> None: + self.settings = settings or get_settings() + self.runtime = runtime or QwenRuntime(self.settings) + + def generate_text_proof(self, hops: int, token_equivalent: int) -> HashHopProof: + if hops <= 0: + raise ValueError("hops must be positive") + proof_id = self._proof_id("text", hops, token_equivalent) + root = self.settings.data_dir / "eval_proofs" / proof_id + root.mkdir(parents=True, exist_ok=True) + chain = [secrets.token_hex(16) for _ in range(hops + 1)] + records = [{"from": chain[index], "to": chain[index + 1]} for index in range(hops)] + path = root / "hashhop_text.jsonl" + path.write_text("\n".join(json.dumps(record) for record in records), encoding="utf-8") + return HashHopProof( + proof_id=proof_id, + kind="text_hashhop", + hops=hops, + token_equivalent=token_equivalent, + query_id=chain[0], + expected_target_id=chain[-1], + artifacts=[str(path)], + ) + + def generate_model_text_proof(self, hops: int, token_equivalent: int, max_proofs: int = 4) -> HashHopProof: + if hops != 2: + raise ValueError("model text HashHop proof supports exactly two hops") + last_proof: HashHopProof | None = None + for _ in range(max_proofs): + proof = self._generate_model_text_proof_once(hops, token_equivalent) + if proof.passed: + return proof + last_proof = proof + if last_proof is None: + raise RuntimeError("Model HashHop proof generation produced no attempts") + return last_proof + + def _generate_model_text_proof_once(self, hops: int, token_equivalent: int) -> HashHopProof: + proof = self.generate_text_proof(hops, token_equivalent) + root = Path(proof.artifacts[0]).parent + chain_path = root / "hashhop_text.jsonl" + try: + chain_text = chain_path.read_text(encoding="utf-8") + except OSError as error: + raise ValueError(f"{chain_path}: the HashHop chain is unreadable ({error})") from error + try: + records = [json.loads(line) for line in chain_text.splitlines() if line.strip()] + except json.JSONDecodeError as error: + raise ValueError(f"{chain_path}: the HashHop chain holds a malformed line ({error})") from error + decoys = [ + {"from": secrets.token_hex(16), "to": secrets.token_hex(16)} + for _ in range(max(4, hops * 2)) + ] + shuffled = records + decoys + random.Random(proof.proof_id).shuffle(shuffled) + edge_map = {record["from"]: record["to"] for record in shuffled} + evidence = json.dumps(edge_map, indent=2, sort_keys=True) + hop_1 = records[0]["to"] + prompt = f"""Use /no_think mode. Return compact JSON only. + +Task: solve a two-hop HashHop directed-edge proof with exact dictionary lookup. + +Example: +edge_map = {{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "cccccccccccccccccccccccccccccccc"}} +start_id = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +hop_1 = edge_map[start_id] +answer = edge_map[hop_1] +JSON = {{"hop_1":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","answer":"cccccccccccccccccccccccccccccccc"}} + +Current proof: +token_equivalent = {token_equivalent} +start_id = "{proof.query_id}" +hops = {hops} +edge_map = {evidence} + +Evaluate: +hop_1 = edge_map["{proof.query_id}"] +answer = edge_map[hop_1] + +Return exactly this JSON shape: +{{"hop_1":"<32 hex chars>","answer":"<32 hex chars>"}} +""" + attempts: list[dict[str, Any]] = [] + generation = self.runtime.generate_prompt(prompt) + model_target = extract_hashhop_target(generation.text) + attempts.append( + { + "step": "two_hop_expression", + "raw_generation": generation.text, + "parsed_target": model_target, + "expected_target": proof.expected_target_id, + "passed": model_target == proof.expected_target_id, + "input_tokens": generation.input_tokens, + "output_tokens": generation.output_tokens, + } + ) + if model_target != proof.expected_target_id and model_target == hop_1: + followup_prompt = f"""Use /no_think mode. Return compact JSON only. + +Task: continue a HashHop proof from a model-derived first hop. + +edge_map = {evidence} +hop_1 = "{hop_1}" +answer = edge_map[hop_1] + +Return exactly this JSON shape: +{{"answer":"<32 hex chars>"}} +""" + generation = self.runtime.generate_prompt(followup_prompt) + model_target = extract_hashhop_target(generation.text) + attempts.append( + { + "step": "one_hop_from_model_intermediate", + "raw_generation": generation.text, + "parsed_target": model_target, + "expected_target": proof.expected_target_id, + "passed": model_target == proof.expected_target_id, + "input_tokens": generation.input_tokens, + "output_tokens": generation.output_tokens, + } + ) + proof.model_target_id = model_target + proof.passed = model_target == proof.expected_target_id + artifact = root / "model_text_hashhop.json" + proof.artifacts.append(str(artifact)) + write_json( + artifact, + { + "proof": proof.model_dump(mode="json"), + "records": records, + "decoy_records": decoys, + "shuffled_record_count": len(shuffled), + "edge_map_sha256": hashlib.sha256(evidence.encode("utf-8")).hexdigest(), + "model_attempts": attempts, + "raw_generation": generation.text, + "parsed_target": model_target, + "expected_target": proof.expected_target_id, + "passed": proof.passed, + "runtime": { + "backend": generation.backend, + "adapter_path": generation.adapter_path, + "adapter_kind": generation.adapter_kind, + "adapter_id": generation.adapter_id, + "adapter_sha256": generation.adapter_sha256, + "input_tokens": generation.input_tokens, + "output_tokens": generation.output_tokens, + }, + }, + ) + (root / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") + return proof + + def generate_model_visual_proof(self, hops: int, token_equivalent: int, max_proofs: int = 3) -> HashHopProof: + if hops not in {1, 2}: + raise ValueError("model visual HashHop proof supports one or two hops") + last_proof: HashHopProof | None = None + for _ in range(max_proofs): + if hops == 1: + proof = self._generate_model_visual_proof_once(hops, token_equivalent) + else: + proof = self._generate_model_visual_two_hop_proof_once(token_equivalent) + if proof.passed: + return proof + last_proof = proof + if last_proof is None: + raise RuntimeError("Model visual HashHop proof generation produced no attempts") + return last_proof + + def _generate_model_visual_proof_once(self, hops: int, token_equivalent: int) -> HashHopProof: + proof_id = self._proof_id("visual-model", hops, token_equivalent) + root = self.settings.data_dir / "eval_proofs" / proof_id + image_dir = root / "visual_blocks" + image_dir.mkdir(parents=True, exist_ok=True) + source_id = secrets.token_hex(16) + target_id = secrets.token_hex(16) + decoy_ids = [secrets.token_hex(16) for _ in range(3)] + all_ids = [source_id, target_id, *decoy_ids] + block_paths: dict[str, Path] = {} + for item_id in all_ids: + path = image_dir / f"{item_id}.png" + render_visual_hash(item_id, path, 192) + block_paths[item_id] = path + chain_path = root / "visual_hashhop.jsonl" + chain_path.write_text(json.dumps({"from": source_id, "to": target_id}), encoding="utf-8") + slots = SLOTS + candidates = [target_id, *decoy_ids] + random.Random(proof_id).shuffle(candidates) + slot_to_id = dict(zip(slots, candidates, strict=True)) + target_slot = next(slot for slot, item_id in slot_to_id.items() if item_id == target_id) + arena_path = root / "visual_hashhop_arena.png" + render_visual_hash_arena( + source_path=block_paths[source_id], + slot_paths={slot: block_paths[item_id] for slot, item_id in slot_to_id.items()}, + output_path=arena_path, + ) + proof = HashHopProof( + proof_id=proof_id, + kind="visual_hashhop", + hops=hops, + token_equivalent=token_equivalent, + query_id=source_id, + expected_target_id=target_id, + artifacts=[*(str(path) for path in block_paths.values()), str(chain_path), str(arena_path)], + ) + prompt = """Use /no_think mode. Return compact JSON only. + +The attached image is a Visual HashHop arena. +The large visual hash block on the left is the source. +Four candidate visual hash blocks are arranged on the right. +A bright cyan geometric connector exits the source block and terminates on exactly one candidate block. +Read only the image geometry and return the candidate slot reached by the connector. +Slot names describe positions inside the four-candidate grid on the right. + +Allowed target_slot values: +top-left, top-right, bottom-left, bottom-right + +Return exactly this JSON shape: +{"target_slot":""} +""" + generation = self.runtime.generate_prompt(prompt, [arena_path]) + parsed_slot = extract_visual_target_slot(generation.text) + model_target = slot_to_id.get(parsed_slot) if parsed_slot else None + proof.model_target_id = model_target + proof.passed = model_target == proof.expected_target_id + artifact = root / "model_visual_hashhop.json" + proof.artifacts.append(str(artifact)) + write_json( + artifact, + { + "proof": proof.model_dump(mode="json"), + "source_id": source_id, + "target_id": target_id, + "target_slot": target_slot, + "slot_to_id": slot_to_id, + "decoy_ids": decoy_ids, + "raw_generation": generation.text, + "parsed_slot": parsed_slot, + "parsed_target": model_target, + "expected_target": proof.expected_target_id, + "passed": proof.passed, + "runtime": { + "backend": generation.backend, + "adapter_path": generation.adapter_path, + "adapter_kind": generation.adapter_kind, + "adapter_id": generation.adapter_id, + "adapter_sha256": generation.adapter_sha256, + "input_tokens": generation.input_tokens, + "output_tokens": generation.output_tokens, + }, + }, + ) + (root / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") + return proof + + def _generate_model_visual_two_hop_proof_once(self, token_equivalent: int) -> HashHopProof: + hops = 2 + proof_id = self._proof_id("visual-model", hops, token_equivalent) + root = self.settings.data_dir / "eval_proofs" / proof_id + image_dir = root / "visual_blocks" + image_dir.mkdir(parents=True, exist_ok=True) + source_id = secrets.token_hex(16) + intermediate_id = secrets.token_hex(16) + target_id = secrets.token_hex(16) + intermediate_decoys = [secrets.token_hex(16) for _ in range(3)] + target_decoys = [secrets.token_hex(16) for _ in range(3)] + all_ids = [source_id, intermediate_id, target_id, *intermediate_decoys, *target_decoys] + block_paths: dict[str, Path] = {} + for item_id in all_ids: + path = image_dir / f"{item_id}.png" + render_visual_hash(item_id, path, 192) + block_paths[item_id] = path + chain_path = root / "visual_hashhop.jsonl" + chain_path.write_text( + "\n".join( + [ + json.dumps({"from": source_id, "to": intermediate_id}), + json.dumps({"from": intermediate_id, "to": target_id}), + ] + ), + encoding="utf-8", + ) + slots = SLOTS + intermediate_candidates = [intermediate_id, *intermediate_decoys] + target_candidates = [target_id, *target_decoys] + random.Random(f"{proof_id}:intermediate").shuffle(intermediate_candidates) + random.Random(f"{proof_id}:target").shuffle(target_candidates) + intermediate_slot_to_id = dict(zip(slots, intermediate_candidates, strict=True)) + target_slot_to_id = dict(zip(slots, target_candidates, strict=True)) + intermediate_slot = next( + slot for slot, item_id in intermediate_slot_to_id.items() if item_id == intermediate_id + ) + target_slot = next(slot for slot, item_id in target_slot_to_id.items() if item_id == target_id) + arena_path = root / "visual_hashhop_two_hop_arena.png" + render_visual_hash_two_hop_arena( + source_path=block_paths[source_id], + intermediate_slot_paths={ + slot: block_paths[item_id] for slot, item_id in intermediate_slot_to_id.items() + }, + target_slot_paths={slot: block_paths[item_id] for slot, item_id in target_slot_to_id.items()}, + output_path=arena_path, + ) + proof = HashHopProof( + proof_id=proof_id, + kind="visual_hashhop", + hops=hops, + token_equivalent=token_equivalent, + query_id=source_id, + expected_target_id=target_id, + artifacts=[*(str(path) for path in block_paths.values()), str(chain_path), str(arena_path)], + ) + prompt = """Use /no_think mode. Return compact JSON only. + +The attached image is a two-hop Visual HashHop arena. +The source visual hash block is on the left. +Four intermediate candidate visual hash blocks are in the center. +Four final candidate visual hash blocks are on the right. +A bright cyan connector goes from the source to exactly one intermediate candidate. +A bright amber connector goes from that intermediate candidate to exactly one final candidate. +Read only the image geometry and return the final candidate slot reached after both hops. +The intermediate_slot value describes the center grid. The target_slot value describes the rightmost final-candidate grid. + +Allowed slot values: +top-left, top-right, bottom-left, bottom-right + +Return exactly this JSON shape: +{"intermediate_slot":"","target_slot":""} +""" + generation = self.runtime.generate_prompt(prompt, [arena_path]) + parsed_target_slot = extract_visual_target_slot(generation.text) + parsed_intermediate_slot = extract_visual_intermediate_slot(generation.text) + model_target = target_slot_to_id.get(parsed_target_slot) if parsed_target_slot else None + proof.model_target_id = model_target + proof.passed = model_target == proof.expected_target_id + artifact = root / "model_visual_hashhop.json" + proof.artifacts.append(str(artifact)) + write_json( + artifact, + { + "proof": proof.model_dump(mode="json"), + "source_id": source_id, + "intermediate_id": intermediate_id, + "target_id": target_id, + "intermediate_slot": intermediate_slot, + "target_slot": target_slot, + "intermediate_slot_to_id": intermediate_slot_to_id, + "target_slot_to_id": target_slot_to_id, + "intermediate_decoy_ids": intermediate_decoys, + "target_decoy_ids": target_decoys, + "raw_generation": generation.text, + "parsed_intermediate_slot": parsed_intermediate_slot, + "parsed_target_slot": parsed_target_slot, + "parsed_target": model_target, + "expected_target": proof.expected_target_id, + "passed": proof.passed, + "runtime": { + "backend": generation.backend, + "adapter_path": generation.adapter_path, + "adapter_kind": generation.adapter_kind, + "adapter_id": generation.adapter_id, + "adapter_sha256": generation.adapter_sha256, + "input_tokens": generation.input_tokens, + "output_tokens": generation.output_tokens, + }, + }, + ) + (root / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") + return proof + + def generate_visual_proof(self, hops: int, token_equivalent: int, tile_size: int = 256) -> HashHopProof: + if hops <= 0: + raise ValueError("hops must be positive") + proof_id = self._proof_id("visual", hops, token_equivalent) + root = self.settings.data_dir / "eval_proofs" / proof_id + image_dir = root / "visual_blocks" + image_dir.mkdir(parents=True, exist_ok=True) + ids = [secrets.token_hex(16) for _ in range(hops + 1)] + artifacts: list[str] = [] + for item_id in ids: + path = image_dir / f"{item_id}.png" + render_visual_hash(item_id, path, tile_size) + artifacts.append(str(path)) + chain_path = root / "visual_hashhop.jsonl" + chain_path.write_text( + "\n".join(json.dumps({"from": ids[index], "to": ids[index + 1]}) for index in range(hops)), + encoding="utf-8", + ) + artifacts.append(str(chain_path)) + return HashHopProof( + proof_id=proof_id, + kind="visual_hashhop", + hops=hops, + token_equivalent=token_equivalent, + query_id=ids[0], + expected_target_id=ids[-1], + artifacts=artifacts, + ) + + def _proof_id(self, kind: str, hops: int, token_equivalent: int) -> str: + raw = f"{kind}:{hops}:{token_equivalent}:{secrets.token_hex(8)}" + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] diff --git a/backend/src/ely_eye/proofs/hashhop_verify.py b/backend/src/ely_eye/proofs/hashhop_verify.py new file mode 100644 index 0000000..395ef61 --- /dev/null +++ b/backend/src/ely_eye/proofs/hashhop_verify.py @@ -0,0 +1,73 @@ +"""Judge a HashHop proof: chain, answer, artifacts, pass rate. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +from ..evaluation_schemas import ( + HashHopProof, +) +from .hashhop import HashHopEvaluator + + +def hashhop_chain_valid(proof: HashHopProof) -> bool: + chain_artifacts = [Path(path) for path in proof.artifacts if path.endswith(".jsonl")] + if len(chain_artifacts) != 1 or not chain_artifacts[0].exists(): + return False + try: + rows = [ + json.loads(line) + for line in chain_artifacts[0].read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (OSError, json.JSONDecodeError): + return False # an unreadable chain is not a valid chain, and this is a predicate + if len(rows) != proof.hops: + return False + if rows[0]["from"] != proof.query_id or rows[-1]["to"] != proof.expected_target_id: + return False + for index in range(1, len(rows)): + if rows[index - 1]["to"] != rows[index]["from"]: + return False + return all(Path(path).exists() for path in proof.artifacts) + + +def copy_proof_artifacts(proof: HashHopProof, target: Path) -> None: + target.mkdir(parents=True, exist_ok=True) + source_roots = {Path(path).parent for path in proof.artifacts} + for source_root in source_roots: + if source_root.name == "visual_blocks": + shutil.copytree(source_root, target / "visual_blocks", dirs_exist_ok=True) + for artifact in proof.artifacts: + path = Path(artifact) + if path.is_file() and path.parent.name != "visual_blocks": + shutil.copy2(path, target / path.name) + (target / "proof.json").write_text(proof.model_dump_json(indent=2), encoding="utf-8") + + +def pass_rate(proofs: list[HashHopProof]) -> float: + if not proofs: + return 0.0 + return sum(1 for proof in proofs if proof.passed) / len(proofs) + + +def collect_model_visual_proofs( + evaluator: HashHopEvaluator, + hops: int, + token_equivalent: int, + attempts: int, +) -> list[HashHopProof]: + """Run exactly `attempts` puzzles and keep every verdict. + + The old loop stopped the moment the running pass rate crossed its target, so a model could be + re-rolled until it passed. A fixed attempt count is what makes the rate mean anything. + """ + if attempts <= 0: + raise ValueError(f"collect needs a positive attempt count, got {attempts}") + return [ + evaluator.generate_model_visual_proof(hops, token_equivalent, max_proofs=5) + for _ in range(attempts) + ] diff --git a/backend/src/ely_eye/proofs/report.py b/backend/src/ely_eye/proofs/report.py new file mode 100644 index 0000000..35680ca --- /dev/null +++ b/backend/src/ely_eye/proofs/report.py @@ -0,0 +1,142 @@ +"""Small readers, writers and diggers the proof checks share. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from pathlib import Path +from typing import Any + +import zstandard as zstd + + +def parquet_row_count(path: Path) -> int: + if not path.exists(): + return -1 + import pyarrow.parquet as pq + + return pq.read_metadata(path).num_rows + + +def compiled_context_digest(context: Any) -> str: + payload = { + "hits": [hit.atom.atom_id for hit in context.hits], + "packed_text": context.packed_text, + "token_equivalent": context.token_equivalent, + } + data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + return hashlib.sha256(data).hexdigest() + + +def read_memory_capsule(path: Path) -> dict[str, Any]: + data = path.read_bytes() + prefix = b"ELYMEMCAP1\n" + if not data.startswith(prefix): + raise ValueError(f"Invalid memory capsule header: {path}") + try: + payload = zstd.ZstdDecompressor().decompress(data[len(prefix) :]) + parsed = json.loads(payload.decode("utf-8")) + except (zstd.ZstdError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"Cannot decode the memory capsule at {path}: {error}") from error + if not isinstance(parsed, dict): + raise ValueError(f"Invalid memory capsule payload: {path}") # noqa: TRY004 # the capsule's own bytes are wrong, not the caller's argument + return parsed + + +def validated_segment_hash(segment: dict[str, Any]) -> str: + expected = segment.get("segment_sha256") + if not isinstance(expected, str): + return "" + payload = {key: value for key, value in segment.items() if key != "segment_sha256"} + actual = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + return actual if actual == expected else "" + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def read_json_or_empty(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + value = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return value if isinstance(value, dict) else {} + + +def format_scores(scores: dict[str, Any]) -> str: + parts: list[str] = [] + for name in sorted(scores): + score = scores[name] + if isinstance(score, (int, float)) and math.isfinite(float(score)): + parts.append(f"{name}={float(score):.2f}") + else: + parts.append(f"{name}=unscored") + return ", ".join(parts) or "none" + + +def is_finite_number(value: Any) -> bool: + return isinstance(value, (int, float)) and math.isfinite(float(value)) + + +def compact_generation_context(context: Any, section_hint: str, max_chars: int = 2400) -> Any: + text_hits = [hit for hit in context.hits if hit.atom.modality.value == "text"] + hits = (text_hits or context.hits)[:1] + sections: list[str] = [] + for hit in hits: + atom = hit.atom + evidence_text = extract_markdown_section(atom.text, section_hint) or atom.text + evidence_text = evidence_text.strip()[:max_chars] + citation = { + "atom_id": atom.atom_id, + "modality": atom.modality.value, + "source": atom.source, + "image_ref": atom.image_ref, + "layout": atom.layout.model_dump() if atom.layout else None, + "score": hit.final_score, + } + sections.append(f"[EVIDENCE]\n{citation}\n{evidence_text}\n[/EVIDENCE]") + packed_text = "\n\n".join(sections) + return context.model_copy( + update={ + "hits": hits, + "packed_text": packed_text, + "token_equivalent": max(1, len(packed_text) // 4), + } + ) + + +def extract_markdown_section(text: str, section_hint: str) -> str: + lines = text.splitlines() + start = None + for index, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith("#") and section_hint in stripped: + start = index + break + if start is None: + return "" + end = len(lines) + start_level = len(lines[start]) - len(lines[start].lstrip("#")) + for index in range(start + 1, len(lines)): + stripped = lines[index].strip() + if stripped.startswith("#"): + level = len(lines[index]) - len(lines[index].lstrip("#")) + if level <= start_level: + end = index + break + return "\n".join(lines[start:end]) + + +def relative_artifacts(root: Path) -> dict[str, str]: + artifacts: dict[str, str] = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + key = path.relative_to(root).as_posix() + artifacts[key] = key + return artifacts diff --git a/backend/src/ely_eye/proofs/repository_checks.py b/backend/src/ely_eye/proofs/repository_checks.py new file mode 100644 index 0000000..65cb947 --- /dev/null +++ b/backend/src/ely_eye/proofs/repository_checks.py @@ -0,0 +1,37 @@ +"""Find the repository the proof suite is allowed to talk about. +""" + +from __future__ import annotations + +import os +from pathlib import Path + + +def project_root() -> Path: + current = Path.cwd().resolve() + for candidate in (current, *current.parents): + if (candidate / "PRD.md").exists() and (candidate / "backend" / "src" / "ely_eye").exists(): + return candidate + return current + + +def discover_repository_files(root: Path) -> list[Path]: + ignored_dirs = {".git", ".ely_eye", ".venv", ".venv-linux", "__pycache__", "node_modules", "dist", "build", "target"} + files: list[Path] = [] + for current, dirs, names in os.walk(root): + dirs[:] = [name for name in dirs if name not in ignored_dirs and not name.startswith(".venv")] + current_path = Path(current) + for name in names: + path = current_path / name + try: + relative = path.relative_to(root) + except ValueError: + continue + if any(part in ignored_dirs or part.startswith(".venv") for part in relative.parts): + continue + try: + path.stat() + except OSError: + continue + files.append(path) + return files diff --git a/backend/src/ely_eye/proofs/suite.py b/backend/src/ely_eye/proofs/suite.py new file mode 100644 index 0000000..f61e64f --- /dev/null +++ b/backend/src/ely_eye/proofs/suite.py @@ -0,0 +1,127 @@ +"""The proof suite: one check per claim, and a verdict that says what it proves. +""" + +from __future__ import annotations + +import hashlib +import secrets +from pathlib import Path +from typing import Literal + +from ..cartridge import CartridgeService +from ..config import Settings, get_settings +from ..db import Database +from ..evaluation_schemas import ( + ProofCheck, + ProofSuiteReport, +) +from ..schemas import ( + CartridgeManifest, + TrainingAdapterKind, +) +from .checks_cartridge import CartridgeChecks +from .checks_runtime import RuntimeChecks +from .checks_training import TrainingChecks +from .checks_visual import VisualChecks +from .report import ( + relative_artifacts, +) + + +class PRDProofSuite(TrainingChecks, CartridgeChecks, RuntimeChecks, VisualChecks): + required_adapters = ( + TrainingAdapterKind.hme_core_lora, + TrainingAdapterKind.hme_vision_lora, + TrainingAdapterKind.hme_ttt_vl, + TrainingAdapterKind.hme_visual_mtp, + TrainingAdapterKind.hme_router, + TrainingAdapterKind.hme_retrieval, + ) + + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or get_settings() + self.db = Database(self.settings) + self.cartridges = CartridgeService(self.settings, self.db) + + def run(self, cartridge_id: str | None = None) -> ProofSuiteReport: + row = self._resolve_cartridge(cartridge_id) + cartridge_id = str(row["cartridge_id"]) + root = Path(row["root_path"]) + manifest = CartridgeManifest.model_validate_json(row["manifest_json"]) + proof_id = self._proof_id(cartridge_id, manifest.dna) + proof_root = self.settings.data_dir / "eval_proofs" / proof_id + proof_root.mkdir(parents=True, exist_ok=True) + dna_before = manifest.dna + + asset_check, manifest = self._cartridge_assets_check(cartridge_id, root, proof_root) + checks = [ + self._adapter_training_check(cartridge_id, root, proof_root), + self._runtime_adapter_check(proof_root), + self._runtime_generation_check(proof_root), + asset_check, + self._memory_capsule_check(root, proof_root), + self._cartridge_replay_check(cartridge_id, root, manifest, proof_root), + self._visual_hashhop_check(proof_root), + self._standard_benchmark_check(proof_root), + self._repository_input_check(proof_root), + self._runtime_profile_check(proof_root), + self._research_module_check(proof_root), + self._learned_memory_recall_check(cartridge_id, root, proof_root), + self._memory_dna_check(root, manifest, proof_root), + self._cache_trace_check(proof_root), + self._visual_contradiction_check(proof_root), + ] + # A deferred or removed check is not a failure and not a pass: the suite is only as green + # as the checks that actually ran. A claim that never ran holds the verdict, because + # "passed" would say the PRD is proven while part of it was never attempted. + failed = [check for check in checks if check.status == "failed"] + unrun = [check for check in checks if check.status in ("deferred", "removed")] + status: Literal["passed", "failed", "deferred"] = ( + "failed" if failed else ("deferred" if unrun else "passed") + ) + report = ProofSuiteReport( + proof_id=proof_id, + cartridge_id=cartridge_id, + status=status, + checks=checks, + artifacts={}, + dna_before=dna_before, + dna_after=dna_before, + ) + report_path = proof_root / "proof_suite.json" + report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") + report.artifacts = relative_artifacts(proof_root) + report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") + attached_manifest = self.cartridges.attach_eval_proof(cartridge_id, proof_root) + report.dna_after = attached_manifest.dna + report_path.write_text(report.model_dump_json(indent=2), encoding="utf-8") + cartridge_report = root / "eval_proofs" / proof_id / "proof_suite.json" + cartridge_report.write_text(report.model_dump_json(indent=2), encoding="utf-8") + return report + + def runtime_generation_proof(self) -> ProofCheck: + proof_id = f"runtime_generation_{secrets.token_hex(8)}" + proof_root = self.settings.data_dir / "eval_proofs" / proof_id + proof_root.mkdir(parents=True, exist_ok=True) + return self._runtime_generation_check(proof_root) + + def visual_contradiction_proof(self) -> ProofCheck: + proof_id = f"visual_contradiction_{secrets.token_hex(8)}" + proof_root = self.settings.data_dir / "eval_proofs" / proof_id + proof_root.mkdir(parents=True, exist_ok=True) + return self._visual_contradiction_check(proof_root) + + def latest(self) -> ProofSuiteReport | None: + proof_root = self.settings.data_dir / "eval_proofs" + reports = sorted( + proof_root.glob("prd_proof_suite_*/proof_suite.json"), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + if not reports: + return None + return ProofSuiteReport.model_validate_json(reports[0].read_text(encoding="utf-8")) + + def _proof_id(self, cartridge_id: str, dna: str | None) -> str: + raw = f"{cartridge_id}:{dna}:{secrets.token_hex(8)}" + return f"prd_proof_suite_{hashlib.sha256(raw.encode('utf-8')).hexdigest()[:16]}" diff --git a/backend/src/ely_eye/proofs/table.py b/backend/src/ely_eye/proofs/table.py new file mode 100644 index 0000000..c5a45c9 --- /dev/null +++ b/backend/src/ely_eye/proofs/table.py @@ -0,0 +1,34 @@ +"""Render a proof report as one row per claim, verdict first. + +A JSON report hides the shape of the evidence: a deferred claim and a passed claim look equally +short. The table puts the counts on the first line and the reason under every claim that did not +pass, so nobody reads an unrun claim as a proven one. +""" + +from __future__ import annotations + +from ..evaluation_schemas import ProofSuiteReport + +STATUS_ORDER = ("failed", "passed", "deferred", "removed") + + +def render_table(report: ProofSuiteReport) -> str: + counts: dict[str, int] = {status: 0 for status in STATUS_ORDER} + for check in report.checks: + counts[check.status] = counts.get(check.status, 0) + 1 + lines = [ + f"proof {report.proof_id} · cartridge {report.cartridge_id} · suite {report.status}", + "counts: " + ", ".join(f"{status} {counts.get(status, 0)}" for status in STATUS_ORDER), + "", + ] + for status in STATUS_ORDER: + for check in report.checks: + if check.status != status: + continue + lines.append(f"[{check.status:^7}] {check.name}") + lines.append(f" ran: {check.detail}") + if check.reason: + lines.append(f" why: {check.reason}") + if check.rerun: + lines.append(f" rerun: {check.rerun}") + return "\n".join(lines) diff --git a/backend/src/ely_eye/proofs/training_trace.py b/backend/src/ely_eye/proofs/training_trace.py new file mode 100644 index 0000000..5a64b1b --- /dev/null +++ b/backend/src/ely_eye/proofs/training_trace.py @@ -0,0 +1,358 @@ +"""Check that an adapter's training artifacts tell a consistent story. +""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +from ..schemas import ( + AdapterManifest, + TrainingAdapterKind, +) +from ..training import ( + AUTOCAST_DTYPE, + GRADIENT_CHECKPOINTING_MODE, + TRAINING_FRAMEWORK, + TRAINING_METHOD, + TRAINING_PRECISION, + TRAINING_PROOF_VERSION, + safetensor_weight_stats, + sha256_file, +) + + +def count_safetensor_keys(path: Path) -> int: + return int(safetensor_weight_stats(path)["tensor_count"]) + + +def peft_config_loads(path: Path) -> bool: + try: + from peft import PeftConfig + + PeftConfig.from_pretrained(str(path)) + return True + except Exception: # noqa: BLE001 # a probe records its failure instead of crashing the proof run + return False + + +def validate_training_contract(manifest: AdapterManifest) -> dict[str, Any]: + target_modules_ok = training_target_modules_ok(manifest.kind, manifest.target_modules) + report = { + "proof_version_ok": manifest.proof_version >= TRAINING_PROOF_VERSION, + "method_ok": manifest.training_method == TRAINING_METHOD, + "precision_ok": manifest.precision == TRAINING_PRECISION and manifest.bf16, + "framework_ok": manifest.framework == TRAINING_FRAMEWORK, + "optimizer_ok": manifest.optimizer == "paged_adamw_8bit", + "optimizer_family_ok": manifest.optimizer_family == "8-bit AdamW / paged optimizer", + "gradient_checkpointing_ok": ( + manifest.gradient_checkpointing + and manifest.gradient_checkpointing_mode == GRADIENT_CHECKPOINTING_MODE + ), + "autocast_dtype_ok": manifest.autocast_dtype == AUTOCAST_DTYPE, + "unsloth_version_ok": bool(manifest.unsloth_version), + "triton_version_ok": bool(manifest.triton_version), + "xformers_version_ok": bool(manifest.xformers_version), + "unsloth_model_type_ok": bool(manifest.unsloth_model_type), + "max_length_ok": manifest.max_length >= 8192, + "target_modules_ok": target_modules_ok, + } + report["valid"] = all(report.values()) + return report + + +def training_target_modules_ok(kind: TrainingAdapterKind, target_modules: list[str]) -> bool: + targets = set(target_modules) + core = {"q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"} + if kind == TrainingAdapterKind.hme_core_lora: + return core.issubset(targets) + if kind == TrainingAdapterKind.hme_vision_lora: + return bool(targets) and any("visual" in target or "vision" in target for target in targets) + if kind == TrainingAdapterKind.hme_ttt_vl: + has_visual = any("visual" in target or "vision" in target for target in targets) + has_mlp = bool({"gate_proj", "up_proj", "down_proj"} & targets) + return has_visual and has_mlp + if kind == TrainingAdapterKind.hme_visual_mtp: + return any("mtp" in target or "draft" in target or target == "lm_head" for target in targets) + if kind in {TrainingAdapterKind.hme_router, TrainingAdapterKind.hme_retrieval}: + return core.issubset(targets) + return False + + +def training_payload_contract_ok(payload: dict[str, Any], manifest: AdapterManifest) -> bool: + return ( + int(payload.get("proof_version") or 0) >= TRAINING_PROOF_VERSION + and payload.get("training_method") == TRAINING_METHOD + and payload.get("precision") == TRAINING_PRECISION + and payload.get("framework") == TRAINING_FRAMEWORK + and payload.get("optimizer_family") == manifest.optimizer_family + and bool(payload.get("gradient_checkpointing")) + and payload.get("gradient_checkpointing_mode") == GRADIENT_CHECKPOINTING_MODE + and payload.get("autocast_dtype") == AUTOCAST_DTYPE + and payload.get("unsloth_version") == manifest.unsloth_version + and payload.get("triton_version") == manifest.triton_version + and payload.get("xformers_version") == manifest.xformers_version + and payload.get("unsloth_model_type") == manifest.unsloth_model_type + ) + + +def validate_training_trace(manifest: AdapterManifest, adapter_root: Path) -> dict[str, Any]: + trace_path = adapter_root / "training_trace.jsonl" + summary_path = adapter_root / "training_summary.json" + report: dict[str, Any] = { + "trace_path": str(trace_path), + "summary_path": str(summary_path), + "trace_exists": trace_path.exists(), + "summary_exists": summary_path.exists(), + "trace_sha256_ok": False, + "summary_sha256_ok": False, + "step_records": 0, + "loss_history_records": len(manifest.loss_history), + "loss_history_ok": False, + "summary_matches_manifest": False, + "summary_contract_ok": False, + "run_start_contract_ok": False, + "run_finish_contract_ok": False, + "cuda_training": False, + "bf16_supported": False, + "bf16": manifest.bf16, + "valid": False, + } + if not trace_path.exists() or not summary_path.exists(): + return report + try: + trace_rows = [ + json.loads(line) + for line in trace_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + summary = json.loads(summary_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return report + + step_rows = [row for row in trace_rows if row.get("event") == "step"] + start_row: dict[str, Any] = next((row for row in trace_rows if row.get("event") == "run_start"), {}) + finish_row: dict[str, Any] = next( + (row for row in reversed(trace_rows) if row.get("event") == "run_finish"), {} + ) + losses = [row.get("loss") for row in step_rows] + finite_losses = all(isinstance(loss, (int, float)) and math.isfinite(float(loss)) for loss in losses) + final_loss_ok = ( + manifest.final_loss is not None + and bool(losses) + and abs(float(losses[-1]) - float(manifest.final_loss)) < 1e-6 + ) + report.update( + { + "trace_sha256_ok": ( + manifest.training_trace_sha256 is not None + and sha256_file(trace_path) == manifest.training_trace_sha256 + ), + "summary_sha256_ok": ( + manifest.training_summary_sha256 is not None + and sha256_file(summary_path) == manifest.training_summary_sha256 + ), + "step_records": len(step_rows), + "loss_history_ok": ( + len(manifest.loss_history) == manifest.max_steps + and finite_losses + and final_loss_ok + ), + "summary_matches_manifest": ( + summary.get("adapter_kind") == manifest.kind.value + and summary.get("base_model") == manifest.base_model + and summary.get("dataset_sha256") == manifest.dataset_sha256 + and int(summary.get("max_steps") or -1) == manifest.max_steps + and int(summary.get("trainable_params") or -1) == manifest.trainable_params + ), + "summary_contract_ok": training_payload_contract_ok(summary, manifest), + "run_start_contract_ok": training_payload_contract_ok(start_row, manifest), + "run_finish_contract_ok": training_payload_contract_ok(finish_row, manifest), + } + ) + environment = summary.get("environment") if isinstance(summary.get("environment"), dict) else {} + device_name = str(environment.get("device_name") or "") + report["cuda_training"] = bool(environment.get("cuda_available")) and "RTX 4090" in device_name + report["bf16_supported"] = bool(environment.get("bf16_supported")) + report["valid"] = ( + report["trace_sha256_ok"] + and report["summary_sha256_ok"] + and report["step_records"] == manifest.max_steps + and report["loss_history_ok"] + and report["summary_matches_manifest"] + and report["summary_contract_ok"] + and report["run_start_contract_ok"] + and report["run_finish_contract_ok"] + and report["cuda_training"] + and report["bf16_supported"] + and manifest.bf16 + ) + return report + + +def validate_training_proof(manifest: AdapterManifest, adapter_root: Path) -> dict[str, Any]: + proof_path = adapter_root / "training_proof.json" + report: dict[str, Any] = { + "proof_path": str(proof_path), + "proof_exists": proof_path.exists(), + "proof_sha256_ok": False, + "hash_changed": False, + "optimizer_update_count": manifest.optimizer_update_count, + "optimizer_update_count_ok": False, + "max_optimizer_update_l2": manifest.max_optimizer_update_l2, + "max_optimizer_update_l2_ok": False, + "loss_delta": manifest.loss_delta, + "loss_delta_finite": False, + "updates_have_gradients": False, + "updates_have_weight_delta": False, + "proof_weight_sha256_ok": False, + "proof_weight_stats_ok": False, + "proof_cuda_bf16": False, + "proof_bf16_supported": False, + "proof_contract_ok": False, + "proof_optimizer_ok": False, + "proof_max_length_ok": False, + "proof_target_modules_ok": False, + "proof_unsloth_runtime_ok": False, + "proof_matches_manifest": False, + "valid": False, + } + if not proof_path.exists(): + return report + try: + proof = json.loads(proof_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return report + + updates = proof.get("optimizer_updates") if isinstance(proof.get("optimizer_updates"), list) else [] + update_l2_values = [ + float(value) + for item in updates + if isinstance(item, dict) and isinstance(value := item.get("update_l2"), (int, float)) + ] + gradient_l2_values = [ + float(value) + for item in updates + if isinstance(item, dict) and isinstance(value := item.get("gradient_l2"), (int, float)) + ] + weight_report = safetensor_weight_stats(adapter_root / "adapter_model.safetensors") + proof_weight_stats = proof.get("adapter_weight_stats") if isinstance(proof.get("adapter_weight_stats"), dict) else {} + proof_environment = proof.get("environment") if isinstance(proof.get("environment"), dict) else {} + report.update( + { + "proof_sha256_ok": ( + manifest.training_proof_sha256 is not None + and sha256_file(proof_path) == manifest.training_proof_sha256 + ), + "hash_changed": ( + bool(manifest.initial_trainable_sha256) + and bool(manifest.final_trainable_sha256) + and manifest.initial_trainable_sha256 != manifest.final_trainable_sha256 + ), + "optimizer_update_count_ok": ( + manifest.optimizer_update_count > 0 + and len(updates) == manifest.optimizer_update_count + ), + "max_optimizer_update_l2_ok": ( + manifest.max_optimizer_update_l2 is not None + and math.isfinite(float(manifest.max_optimizer_update_l2)) + and float(manifest.max_optimizer_update_l2) > 0.0 + ), + "loss_delta_finite": ( + manifest.loss_delta is not None + and math.isfinite(float(manifest.loss_delta)) + ), + "updates_have_gradients": bool(gradient_l2_values) + and all(math.isfinite(value) and value > 0.0 for value in gradient_l2_values), + "updates_have_weight_delta": bool(update_l2_values) + and all(math.isfinite(value) and value > 0.0 for value in update_l2_values), + "proof_weight_sha256_ok": ( + isinstance(proof.get("adapter_weight_sha256"), str) + and proof.get("adapter_weight_sha256") == weight_report["sha256"] + ), + "proof_weight_stats_ok": ( + proof_weight_stats.get("sha256") == weight_report["sha256"] + and proof_weight_stats.get("tensor_count") == weight_report["tensor_count"] + and proof_weight_stats.get("lora_tensor_count") == weight_report["lora_tensor_count"] + and proof_weight_stats.get("total_elements") == weight_report["total_elements"] + and proof_weight_stats.get("nonzero_elements") == weight_report["nonzero_elements"] + and int(proof_weight_stats.get("lora_tensor_count") or 0) > 0 + and int(proof_weight_stats.get("nonzero_elements") or 0) > 0 + and bool(proof_weight_stats.get("all_finite")) + ), + "proof_cuda_bf16": ( + bool(proof.get("bf16")) + and bool(proof_environment.get("cuda_available")) + and "RTX 4090" in str(proof_environment.get("device_name") or "") + ), + "proof_bf16_supported": bool(proof_environment.get("bf16_supported")), + "proof_contract_ok": training_payload_contract_ok(proof, manifest), + "proof_optimizer_ok": proof.get("optimizer") == "paged_adamw_8bit", + "proof_max_length_ok": int(proof.get("max_length") or 0) >= 8192, + "proof_target_modules_ok": training_target_modules_ok( + manifest.kind, + proof.get("target_modules") if isinstance(proof.get("target_modules"), list) else [], + ), + "proof_unsloth_runtime_ok": ( + bool(proof.get("unsloth_version")) + and bool(proof.get("triton_version")) + and bool(proof.get("xformers_version")) + and bool(proof.get("unsloth_model_type")) + ), + "proof_matches_manifest": ( + proof.get("adapter_kind") == manifest.kind.value + and proof.get("proof_version") == manifest.proof_version + and proof.get("training_method") == manifest.training_method + and proof.get("precision") == manifest.precision + and proof.get("framework") == manifest.framework + and proof.get("base_model") == manifest.base_model + and proof.get("dataset_sha256") == manifest.dataset_sha256 + and proof.get("sample_count") == manifest.sample_count + and proof.get("max_steps") == manifest.max_steps + and proof.get("max_length") == manifest.max_length + and proof.get("rank") == manifest.rank + and proof.get("alpha") == manifest.alpha + and proof.get("target_modules") == manifest.target_modules + and proof.get("optimizer") == manifest.optimizer + and proof.get("optimizer_family") == manifest.optimizer_family + and proof.get("learning_rate") == manifest.learning_rate + and proof.get("gradient_accumulation_steps") == manifest.gradient_accumulation_steps + and proof.get("gradient_checkpointing") == manifest.gradient_checkpointing + and proof.get("gradient_checkpointing_mode") == manifest.gradient_checkpointing_mode + and proof.get("autocast_dtype") == manifest.autocast_dtype + and proof.get("unsloth_version") == manifest.unsloth_version + and proof.get("triton_version") == manifest.triton_version + and proof.get("xformers_version") == manifest.xformers_version + and proof.get("unsloth_model_type") == manifest.unsloth_model_type + and proof.get("bf16") == manifest.bf16 + and proof.get("trainable_params") == manifest.trainable_params + and proof.get("total_params") == manifest.total_params + and proof.get("initial_trainable_sha256") == manifest.initial_trainable_sha256 + and proof.get("final_trainable_sha256") == manifest.final_trainable_sha256 + and proof.get("optimizer_update_count") == manifest.optimizer_update_count + and proof.get("max_optimizer_update_l2") == manifest.max_optimizer_update_l2 + ), + } + ) + report["valid"] = ( + report["proof_sha256_ok"] + and report["hash_changed"] + and report["optimizer_update_count_ok"] + and report["max_optimizer_update_l2_ok"] + and report["loss_delta_finite"] + and report["updates_have_gradients"] + and report["updates_have_weight_delta"] + and report["proof_weight_sha256_ok"] + and report["proof_weight_stats_ok"] + and report["proof_cuda_bf16"] + and report["proof_bf16_supported"] + and report["proof_contract_ok"] + and report["proof_optimizer_ok"] + and report["proof_max_length_ok"] + and report["proof_target_modules_ok"] + and report["proof_unsloth_runtime_ok"] + and report["proof_matches_manifest"] + ) + return report diff --git a/backend/src/ely_eye/text_index.py b/backend/src/ely_eye/text_index.py index 5731e6b..6385d9f 100644 --- a/backend/src/ely_eye/text_index.py +++ b/backend/src/ely_eye/text_index.py @@ -38,11 +38,19 @@ def fts_query(query: str) -> str: A CJK run becomes a quoted phrase so its characters stay adjacent: unphrased, a query for 钥匙 would match text that mentions 钥 and 匙 worlds apart. Latin tokens get a prefix match, which is what lets `atom` reach `atoms`. + + Latin tokens are OR-ed, not AND-ed. A question is longer than the sentence that answers it — + "Which PRD requirements prove long context memory, HashHop, and adapters?" shares only some of + its words with any single atom — and an AND over every word answers such a question with + nothing. bm25 then ranks the candidates by how many of those words each one carries, so the + broadened query does not mean a diluted answer. """ terms = _bare(query).split() if not terms: raise ValueError(f"search needs at least one searchable word, got {query!r}") - return " ".join(_cjk_phrase(term) if _is_cjk(term) else f"{_quote(term)}*" for term in terms) + if len(terms) == 1: + return _cjk_phrase(terms[0]) if _is_cjk(terms[0]) else f"{_quote(terms[0])}*" + return " OR ".join(_cjk_phrase(term) if _is_cjk(term) else f"{_quote(term)}*" for term in terms) def _bare(text: str) -> str: diff --git a/backend/tests/test_proofs.py b/backend/tests/test_proofs.py new file mode 100644 index 0000000..754a02c --- /dev/null +++ b/backend/tests/test_proofs.py @@ -0,0 +1,282 @@ +"""A proof check reports what it measured: passed, failed, deferred, or removed.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import pytest +from PIL import Image + +from ely_eye.cartridge import CartridgeService +from ely_eye.config import Settings +from ely_eye.memory import MemoryService +from ely_eye.proofs import PRDProofSuite +from ely_eye.proofs.arena import ( + _slot_boxes, + render_visual_hash, + render_visual_hash_arena, +) +from ely_eye.proofs.checks_common import _deferred_without_model +from ely_eye.proofs.contradictions import ( + PACKAGED_CORPUS, + build_contradiction_scenarios, + load_corpus, +) +from ely_eye.proofs.table import render_table +from ely_eye.schemas import MemoryOrigin + +# The replay question is fixed inside the suite, so the fixture memory has to answer it in words +# the ranker can match. +PRD_MEMORY = "PRD requirements prove long context memory, HashHop proof, and adapter training." +OTHER_KEYS = "PRD requirements also prove cache invalidation for HashHop proof and adapters." + + +class Harness: + """A suite wired to one temporary store, plus the directory its artifacts land in.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.memory = MemoryService(settings=settings) + self.db = self.memory.db + self.cartridges = CartridgeService(settings=settings, database=self.db) + self.suite = PRDProofSuite(settings) + self.proof_root = settings.data_dir / "eval_proofs" / "test" + self.proof_root.mkdir(parents=True, exist_ok=True) + + def remember(self, text: str) -> str: + atom = self.memory.remember(text, holder_id="wife", origin=MemoryOrigin.self_, scenario="home") + return atom.atom_id + + def cartridge(self, name: str, atom_ids: list[str]) -> str: + atoms = self.db.get_atoms(atom_ids) + sources = self.db.get_sources(sorted({atom.source_id for atom in atoms})) + return self.cartridges.materialize(name, atoms, sources).cartridge_id + + +@pytest.fixture +def harness(tmp_path: Path, model_dir: Path) -> Harness: + return Harness(Settings(home=tmp_path / "home", model_dir=model_dir, default_holder="wife")) + + +def test_a_store_with_nothing_to_ask_fails_the_cache_trace_check(harness: Harness) -> None: + check = harness.suite._cache_trace_check(harness.proof_root) + + assert check.status == "failed" + assert "empty store" in check.reason + + +def test_the_cache_trace_check_requires_a_measured_miss_then_hit(harness: Harness) -> None: + harness.remember(PRD_MEMORY) + + check = harness.suite._cache_trace_check(harness.proof_root) + + assert check.status == "passed", check.detail + assert check.reason == "" + + +def test_the_research_module_check_is_removed_because_it_could_not_fail( + harness: Harness, +) -> None: + check = harness.suite._research_module_check(harness.proof_root) + + assert check.status == "removed" + assert check.reason and check.rerun + + +def test_the_repository_check_fails_when_the_repository_never_reached_the_store( + harness: Harness, +) -> None: + check = harness.suite._repository_input_check(harness.proof_root) + + assert check.status == "failed" + assert "store" in check.detail or "store" in check.reason + + +def test_the_replay_check_answers_only_from_the_cartridge_it_mounts(harness: Harness) -> None: + inside = harness.remember(PRD_MEMORY) + # A second memory that answers the same question but stays out of the cartridge: an unmounted + # replay against the live store would have found it and called that a cartridge replay. + harness.remember(OTHER_KEYS) + cartridge_id = harness.cartridge("keys", [inside]) + manifest = next( + item for item in harness.cartridges.list_manifests() if item.cartridge_id == cartridge_id + ) + + check = harness.suite._cartridge_replay_check( + cartridge_id, harness.settings.cartridge_dir / cartridge_id, manifest, harness.proof_root + ) + + assert check.status == "passed", check.detail + assert harness.cartridges.mount_state().mounted == [] + + +def test_a_check_defers_loudly_when_the_model_it_needs_is_missing( + tmp_path: Path, model_dir: Path +) -> None: + without = _deferred_without_model( + Settings(home=tmp_path / "a"), + tmp_path / "a" / "artifact.json", + "Runtime Adapter Binding", + "PRD 18 requires the trained adapter to run.", + "ely-eye proof runtime with model_dir set", + ) + with_model = _deferred_without_model( + Settings(home=tmp_path / "b", model_dir=model_dir), + tmp_path / "b" / "artifact.json", + "Runtime Adapter Binding", + "PRD 18 requires the trained adapter to run.", + "ely-eye proof runtime with model_dir set", + ) + + assert without is not None and without.status == "deferred" + assert without.rerun == "ely-eye proof runtime with model_dir set" + assert (tmp_path / "a" / "artifact.json").exists() + assert with_model is None + + +def test_the_table_shows_the_verdict_counts_and_the_reason_for_each_claim() -> None: + from ely_eye.evaluation_schemas import ProofCheck, ProofSuiteReport + + report = ProofSuiteReport( + proof_id="proof-1", + cartridge_id="cart-1", + status="deferred", + artifacts={}, + checks=[ + ProofCheck(name="Ran", requirement="r", status="passed", evidence=[], detail="measured x"), + ProofCheck( + name="Waiting", + requirement="r", + status="deferred", + evidence=[], + detail="nothing ran", + reason="no model weights", + rerun="ely-eye proof suite", + ), + ], + ) + + table = render_table(report) + + assert "counts: failed 0, passed 1, deferred 1, removed 0" in table + assert "why: no model weights" in table + assert "rerun: ely-eye proof suite" in table + assert table.index("[passed ") < table.index("[deferred") + assert "[failed" not in table and "[removed" not in table + + +def test_the_latest_command_prints_the_table_and_exits_by_verdict( + tmp_path: Path, monkeypatch +) -> None: + import json + + from typer.testing import CliRunner + + from ely_eye import cli as cli_module + from ely_eye.evaluation_schemas import ProofCheck, ProofSuiteReport + + settings = Settings(home=tmp_path / "home") + monkeypatch.setattr(cli_module, "get_settings", lambda: settings) + proof_root = settings.data_dir / "eval_proofs" + ran = ProofCheck(name="Ran", requirement="r", status="passed", evidence=[], detail="ok") + cases: list[tuple[str, Literal["failed", "deferred"]]] = [ + ("prd_proof_suite_old", "failed"), + ("prd_proof_suite_new", "deferred"), + ] + for proof_id, status in cases: + report = ProofSuiteReport( + proof_id=proof_id, + cartridge_id="cart-1", + status=status, + artifacts={}, + checks=[ran], + ) + directory = proof_root / proof_id + directory.mkdir(parents=True) + (directory / "proof_suite.json").write_text(report.model_dump_json(), encoding="utf-8") + + table = CliRunner().invoke(cli_module.app, ["latest-proof-suite"]) + machine = CliRunner().invoke(cli_module.app, ["latest-proof-suite", "--json"]) + + assert table.exit_code == 0 + assert "proof prd_proof_suite_new" in table.stdout + assert json.loads(machine.stdout)["proof_id"] == "prd_proof_suite_new" + + (proof_root / "prd_proof_suite_new" / "proof_suite.json").write_text( + ProofSuiteReport( + proof_id="prd_proof_suite_new", + cartridge_id="cart-1", + status="failed", + artifacts={}, + checks=[ran], + ).model_dump_json(), + encoding="utf-8", + ) + failing = CliRunner().invoke(cli_module.app, ["latest-proof-suite"]) + + assert failing.exit_code == 1 + + monkeypatch.setattr(cli_module, "get_settings", lambda: Settings(home=tmp_path / "empty")) + nothing = CliRunner().invoke(cli_module.app, ["latest-proof-suite"]) + + assert nothing.exit_code != 0 + assert "no proof suite" in nothing.stdout + nothing.stderr # says so instead of printing null + + +def test_the_packaged_corpus_matches_the_scenarios_it_was_written_from() -> None: + scenarios, digest = load_corpus(PACKAGED_CORPUS) + built = build_contradiction_scenarios() + + assert [item["name"] for item in scenarios] == [item["name"] for item in built] + assert [item["expect_drift"] for item in scenarios] == [ + item["expect_drift"] for item in built + ] + assert len(scenarios) >= 10 and digest + + +def test_the_contradiction_check_defers_when_no_corpus_exists_and_names_the_file( + harness: Harness, monkeypatch +) -> None: + from ely_eye.proofs import checks_visual as visual_module + + monkeypatch.setattr(visual_module, "corpus_path", lambda data_dir: None) + + check = harness.suite._visual_contradiction_check(harness.proof_root) + + assert check.status == "deferred" + assert "contradiction_scenarios.json" in check.rerun + assert check.detail == "no contradiction corpus on disk" + + +def test_the_contradiction_check_records_which_corpus_it_graded_against( + harness: Harness, +) -> None: + import json + + check = harness.suite._visual_contradiction_check(harness.proof_root) + artifact = json.loads( + (harness.proof_root / "visual_contradiction_lens.json").read_text(encoding="utf-8") + ) + + assert check.status == "deferred" # graded on our own corpus: reported, not claimed + assert artifact["corpus_sha256"] + assert len(artifact["metrics"]["detected_categories"]) >= 5, artifact["metrics"] + + +def test_the_hashhop_arena_marks_no_candidate_as_the_answer(harness: Harness, tmp_path: Path) -> None: + boxes = _slot_boxes(520, 704, 76, 324, 160) + tiles = {} + for slot in boxes: + path = tmp_path / f"{slot}.png" + render_visual_hash(slot, path, 160) + tiles[slot] = path + output = tmp_path / "arena.png" + + render_visual_hash_arena(tiles["top-left"], tiles, output) + + # Every candidate box carries the same outline: a ring, an arrow, or a highlighted slot would + # tell a model the answer without reading a pixel of the source tile. + image = Image.open(output).convert("RGB") + border_colors = {image.getpixel((box[0] + 2, box[1] + 40)) for box in boxes.values()} + assert border_colors == {(180, 190, 205)} diff --git a/backend/tests/test_text_index.py b/backend/tests/test_text_index.py index 7508c99..f035b8e 100644 --- a/backend/tests/test_text_index.py +++ b/backend/tests/test_text_index.py @@ -24,6 +24,16 @@ def _hits(connection: sqlite3.Connection, query: str) -> list[str]: return [row[0] for row in connection.execute("SELECT original FROM fts WHERE fts MATCH ?", (fts_query(query),))] +def _ranked(connection: sqlite3.Connection, query: str) -> list[str]: + """Matches in bm25 order: the answer that covers more of the question should come first.""" + return [ + row[0] + for row in connection.execute( + "SELECT original FROM fts WHERE fts MATCH ? ORDER BY rank", (fts_query(query),) + ) + ] + + def test_index_text_splits_a_chinese_sentence_into_characters() -> None: assert index_text("我们的钥匙放在玄关抽屉里") == "我 们 的 钥 匙 放 在 玄 关 抽 屉 里" @@ -40,7 +50,28 @@ def test_index_text_folds_full_width_letters_that_search_would_miss() -> None: def test_fts_query_phrases_cjk_and_prefixes_latin() -> None: assert fts_query("钥匙") == '"钥 匙"' - assert fts_query("search atoms") == '"search"* "atoms"*' + assert fts_query("search atoms") == '"search"* OR "atoms"*' + + +def test_a_long_question_reaches_the_answer_that_covers_most_of_its_words() -> None: + """AND over every word answers a long question with nothing; bm25 picks the best candidate.""" + connection = _fts( + [ + "PRD requirements prove long context memory, HashHop proof, and adapter training.", + "The adapter was trained for 29 steps on 37 samples.", + "Unrelated notes about the weather.", + ] + ) + question = "Which Ely-Eye PRD requirements prove long context memory, HashHop, and adapters?" + hits = [ + row[0] + for row in connection.execute( + "SELECT original FROM fts WHERE fts MATCH ? ORDER BY rank", (fts_query(question),) + ) + ] + + assert hits[0].startswith("PRD requirements prove") + assert "Unrelated notes about the weather." not in hits def test_fts_query_refuses_a_query_with_nothing_to_match() -> None: @@ -70,6 +101,10 @@ def test_a_prefix_reaches_the_word_it_grew_into() -> None: assert _hits(connection, "forg") == ["the atom was forgotten"] -def test_a_mixed_query_needs_both_halves_in_the_same_memory() -> None: +def test_a_mixed_query_ranks_the_memory_holding_both_halves_first() -> None: + """A query with a Chinese half and an English half must prefer the memory holding both.""" connection = _fts(["钥匙放在 drawer 里", "钥匙在玄关", "the drawer is empty"]) - assert _hits(connection, "钥匙 drawer") == ["钥匙放在 drawer 里"] + ranked = _ranked(connection, "钥匙 drawer") + + assert ranked[0] == "钥匙放在 drawer 里" + assert set(ranked) == {"钥匙放在 drawer 里", "钥匙在玄关", "the drawer is empty"}