From 181f6bf57e4456ced32c778a2137aaa698bd19eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Wed, 7 Oct 2026 14:30:22 -0400 Subject: [PATCH] Give every memory a holder, an origin, a time and a scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An EvidenceAtom could say what was remembered and nothing else: not whose memory it is, not where it came from, not when it was true, not in what situation it may be used. The baseline measured the cost of that: the same ranker scores recall@10 0.412 over the whole corpus and 0.866 inside one holder's memory, so the missing holder column is worth more than any ranking change in the ablation. - schemas.py: MemoryOrigin, TimePrecision, Visibility; EvidenceAtom gains holder_id, origin, scenario, event_time / event_time_end / event_time_precision / event_time_source, valid_from / valid_to, supersedes, visibility, retention_until. `time` is renamed `ingested_at` — ingestion time gets one name, not two. A validator refuses valid_to before valid_from, event_time_end before event_time, and an atom that supersedes itself. - migrations.py: schema v2 with PRAGMA user_version. A first-version database is upgraded in place (ALTER + RENAME COLUMN), rows intact, and a database newer than this build refuses to open instead of being silently downgraded. - v_atoms_current hides memories that are closed, lapsed or not yet begun; unknown bounds read as visible. Time travel is db.atoms_as_of(moment, holder) because a SQLite view cannot take a parameter. - evaluation_schemas.py: the proof and benchmark report shapes move out of schemas.py to keep it under the file-size limit; importers follow. - backend/tests/test_memory_schema.py: 8 tests, including the v1 -> v2 upgrade and the validity timeline. Verification of record: pytest backend/tests bench/tests -q -> 70 passed; ruff check on every touched file -> all checks passed; mypy (fresh cache) on db.py, migrations.py, schemas.py, evaluation_schemas.py and backend/tests -> no issues. evals.py carries 40 pre-existing mypy errors at HEAD and 40 after this change (measured on both copies); its pyright findings are the same debt, and p0-6 rewrites that file. `peft` and `pyarrow.parquet` "could not be resolved" is the documented pyright artifact: pi-lens runs it with 0 config files, and both import at runtime. --- CHECKLIST.md | 23 +- backend/src/ely_eye/api.py | 16 +- backend/src/ely_eye/benchmarks.py | 14 +- backend/src/ely_eye/db.py | 207 ++++++++-------- backend/src/ely_eye/evals.py | 58 +++-- backend/src/ely_eye/evaluation_schemas.py | 122 ++++++++++ backend/src/ely_eye/migrations.py | 183 ++++++++++++++ backend/src/ely_eye/schemas.py | 201 +++++----------- backend/tests/test_memory_schema.py | 224 +++++++++++++++++ bench/reports/BASELINE_EXTERNAL.md | 64 +++++ bench/reports/baseline_external.json | 280 +++++++++++++++++++--- bench/run_external.py | 27 ++- 12 files changed, 1100 insertions(+), 319 deletions(-) create mode 100644 backend/src/ely_eye/evaluation_schemas.py create mode 100644 backend/src/ely_eye/migrations.py create mode 100644 backend/tests/test_memory_schema.py create mode 100644 bench/reports/BASELINE_EXTERNAL.md diff --git a/CHECKLIST.md b/CHECKLIST.md index 90748b2..f1dfdc5 100644 --- a/CHECKLIST.md +++ b/CHECKLIST.md @@ -33,8 +33,13 @@ - 验收：`python -m bench.run --config bench/configs/ablation.json` 出表（`--config` 已接，配置优先于默认、命令行仍可覆盖）；每格带 n 与 Wilson 95% 下界；`bench/reports/baseline_*.json` 入库 - [ ] **b-4** 外部基线 - [x] b-4.1 仓库内 Mem0/Zep 风格同构对照：扁平 store + holder 作用域 + 无失效（`bench/counterpart.py`：`split_items` 400 字切句、`(holder, sha256(text))` 去重＝Mem0、`supersede` 按 (holder, subject) 时间序关闭旧条目＝Zep；`visible(holder, as_of)` 只对 temporal store 生效）+ `bench/tests/test_counterpart.py` 11 条 - - [~] b-4.2 同语料同硬件跑，出对照数字：`bench/run_external.py` 全量在跑（165,973 atom → 403,859 条 memory item，Zep 式 `supersede` 关闭 27,072 条），数字待 `bench/reports/baseline_external.json` - - [ ] b-4.3 附录：真 Mem0 跑法与参考值，标明其 `add()` 依赖 LLM 的不可比因素 + - [x] b-4.2 同语料同硬件跑，出对照数字：`bench/reports/baseline_external.json` + `BASELINE_EXTERNAL.md`（165,973 atom → 403,859 条 memory item，Zep 式 `supersede` 关闭 27,072 条；5,308 query 全量） + - `mem0_flat [holder scope]` recall@10 **0.866**（Wilson 下界 0.855，MRR 0.629，p50 0.084 s） + - `mem0_flat [whole corpus]` recall@10 **0.412**（p50 0.090 s） + - `zep_temporal [holder scope]` 0.866；`zep_temporal [whole corpus]` 0.429 + - 读法：同一个 ranker、同一批 item，只把候选集从全库换成「这个 holder 的记忆」，recall@10 从 0.412 涨到 0.866（+0.454）——**收益来自 holder 边界，不是更好的排序**；Zep 式失效只在跨 holder 时有小幅正效应（conversation +0.091），在 holder 作用域内反而让 conversation 从 0.993 掉到 0.978、并把 5.8% 的不可答题变成拒答 + - 交叉校验：counterpart 的 whole-corpus 纯稠密 0.412 ≈ 产品 `dense_only` 0.424，两套 harness 互相印证 + - [x] b-4.3 附录：真 Mem0 跑法与参考值，标明其 `add()` 依赖 LLM 的不可比因素（`bench/reports/EXTERNAL_APPENDIX.md`：Mem0 论文 LOCOMO 判分表 full-context 72.90 / Mem0^g 68.44 / Mem0 66.88 / Zep 65.99 / best RAG 60.97，附 token 预算与 p50/p95 延迟；四条不可比理由，其中关键一条是论文只报 search/answer 延迟、不报 `add()` 成本，Zep 的图是异步构建的） - 验收：`bench/reports/baseline_external_*.json`；对照基线与 ely-eye 同标注集同指标 - [x] **b-5** 反自证审计（`bench/audit.py` + `bench/tests/test_audit.py` 9 条） - [x] b-5.1 单次尝试：禁 best-of-N 重摇 —— AST 规则：`while` 循环的测试或 `break` 守卫里出现分数词（pass_rate/target/recall/accuracy/score/hit_rate）即违规；函数名含 `best_of/reroll/until_pass` 即违规。已抓到产品现行违规 `backend/src/ely_eye/evals.py:2351`（Visual HashHop 达标即停） @@ -48,13 +53,13 @@ ## 阶段 1 — P0：让它成为记忆 -- [ ] **p0-1** 记忆模型加四维度 - - [ ] p0-1.1 `holder_id` + `origin`（self / observed / artifact / third_party）+ 写入方不可伪造 - - [ ] p0-1.2 双时间轴：`event_time` / `event_time_end` / `event_time_precision` / `event_time_source` + `valid_from` / `valid_to` + `supersedes` - - [ ] p0-1.3 `scenario` + `visibility`（holder / visible_to / purpose / retention_until） - - [ ] p0-1.4 `PRAGMA user_version` 迁移框架 + 旧库可重放 - - [ ] p0-1.5 视图 `v_atoms_current` 与 `v_atoms_as_of(:ts)` - - 验收：`pytest tests/test_memory_schema.py tests/test_migrations.py` 绿；时间旅行返回历史版本 +- [x] **p0-1** 记忆模型加四维度 + - [x] p0-1.1 `holder_id` + `origin`（self / observed / artifact / third_party）——字段与 DB CHECK 已上；「写入方不可伪造」是服务端盖章，归 p0-2.3 + - [x] p0-1.2 双时间轴：`event_time` / `event_time_end` / `event_time_precision` / `event_time_source` + `valid_from` / `valid_to` + `supersedes`（旧 `time` 列重命名为 `ingested_at`，只有一个名字） + - [x] p0-1.3 `scenario` + `visibility`（scope=holder / visible_to / everyone + holders + purpose）+ `retention_until`（提升为真列，因为 current 视图要过滤它） + - [x] p0-1.4 `PRAGMA user_version` 迁移框架 + 旧库可重放（`migrations.py`：v1 库原地 ALTER + RENAME COLUMN 升到 v2，行不丢；版本高于本构建则 SystemExit） + - [x] p0-1.5 视图 `v_atoms_current` + 时间旅行查询——偏差记录：SQLite 视图不能带参数，所以 as-of 是 `db.atoms_as_of(moment, holder=None)`，谓词与视图一致但去掉 retention（retention 是保存策略，不是当时是否为真） + - 验收：`pytest backend/tests/test_memory_schema.py` 绿（8 条，含 v1→v2 升级不丢行、时间旅行返回历史版本）；实际路径是 `backend/tests/`，不是 `tests/` - [ ] **p0-2** 写入路径 - [ ] p0-2.1 `remember` / `forget` / `supersede` 三写，Python 单一写者 - [ ] p0-2.2 MCP 工具面 + HTTP `POST /api/memory{,/forget,/supersede}` + CLI 三面同源 diff --git a/backend/src/ely_eye/api.py b/backend/src/ely_eye/api.py index f760c33..0ac71a9 100644 --- a/backend/src/ely_eye/api.py +++ b/backend/src/ely_eye/api.py @@ -24,6 +24,15 @@ from .config import Settings, get_settings from .db import Database from .embeddings import EmbeddingService from .evals import HashHopEvaluator, PRDProofSuite +from .evaluation_schemas import ( + BenchmarkPredictionReport, + BenchmarkRegistryReport, + BenchmarkSampleReport, + BenchmarkSourceReport, + HashHopProof, + ProofCheck, + ProofSuiteReport, +) from .evidence_graph import EvidenceGraphService from .ingestion import IngestionService from .profiles import runtime_profile_report @@ -33,16 +42,9 @@ from .schemas import ( AdapterManifest, AdapterStatus, AnswerResponse, - BenchmarkRegistryReport, - BenchmarkPredictionReport, - BenchmarkSampleReport, - BenchmarkSourceReport, CartridgeAssetReport, - HashHopProof, MemoryStatus, PathIngestRequest, - ProofCheck, - ProofSuiteReport, RuntimeProfile, RuntimeProfileReport, SystemStatus, diff --git a/backend/src/ely_eye/benchmarks.py b/backend/src/ely_eye/benchmarks.py index e219754..0ae76ec 100644 --- a/backend/src/ely_eye/benchmarks.py +++ b/backend/src/ely_eye/benchmarks.py @@ -8,7 +8,7 @@ from ast import literal_eval from pathlib import Path from typing import Any -from .schemas import ( +from .evaluation_schemas import ( BenchmarkPredictionReport, BenchmarkPredictionStatus, BenchmarkRegistryReport, @@ -19,7 +19,6 @@ from .schemas import ( BenchmarkSpec, ) - STANDARD_BENCHMARKS = ( BenchmarkSpec( name="MMMU-Pro", @@ -125,7 +124,7 @@ def benchmark_source_report(timeout: int = 20) -> BenchmarkSourceReport: sibling_count=len(siblings), ) ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run sources.append( BenchmarkSourceStatus( name=benchmark.name, @@ -217,7 +216,7 @@ def load_benchmark_sample(benchmark: BenchmarkSpec, output_root: Path) -> Benchm sample_sha256=sha256_json(normalized), media_artifacts=media_artifacts, ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run message = str(exc).splitlines()[0] status = "gated" if "gated dataset" in message.lower() or "authenticated" in message.lower() else "failed" return BenchmarkSampleStatus( @@ -274,7 +273,7 @@ def predict_benchmark_sample(benchmark: BenchmarkSpec, output_root: Path, runtim output_tokens=generation.output_tokens, media_artifacts=prediction_media, ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run message = str(exc).splitlines()[0] status = "gated" if "gated dataset" in message.lower() or "authenticated" in message.lower() else "failed" config, split = BENCHMARK_SAMPLE_LOADS.get(benchmark.name, (None, None)) @@ -351,7 +350,10 @@ def load_records(path: Path) -> list[dict[str, Any]]: with path.open("r", encoding="utf-8") as handle: return [json.loads(line) for line in handle if line.strip()] if suffix == ".json": - data = json.loads(path.read_text(encoding="utf-8")) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{path}: cannot read the benchmark file ({error})") from error if isinstance(data, list): return [dict(item) for item in data if isinstance(item, dict)] if isinstance(data, dict): diff --git a/backend/src/ely_eye/db.py b/backend/src/ely_eye/db.py index 2431e05..18e78ba 100644 --- a/backend/src/ely_eye/db.py +++ b/backend/src/ely_eye/db.py @@ -2,14 +2,15 @@ from __future__ import annotations import json import sqlite3 -from collections.abc import Iterable +from collections.abc import Iterator from contextlib import contextmanager from datetime import datetime from pathlib import Path from typing import Any from .config import Settings, get_settings -from .schemas import EvidenceAtom, SourceRecord +from .migrations import apply as apply_migrations +from .schemas import EvidenceAtom, MemoryOrigin, SourceRecord, TimePrecision, utc_now def _json(value: Any) -> str: @@ -19,7 +20,16 @@ def _json(value: Any) -> str: def _loads(value: str | None, default: Any) -> Any: if value is None or value == "": return default - return json.loads(value) + try: + return json.loads(value) + except json.JSONDecodeError as error: + raise ValueError(f"stored JSON is unreadable: {error}") from error + + +def _datetime(value: str | None) -> datetime | None: + if value is None or value == "": + return None + return datetime.fromisoformat(value) class Database: @@ -30,7 +40,7 @@ class Database: self.init() @contextmanager - def connect(self) -> Iterable[sqlite3.Connection]: + def connect(self) -> Iterator[sqlite3.Connection]: connection = sqlite3.connect(self.path) connection.row_factory = sqlite3.Row connection.execute("PRAGMA foreign_keys = ON") @@ -43,76 +53,7 @@ class Database: def init(self) -> None: with sqlite3.connect(self.path) as connection: - connection.executescript( - """ - PRAGMA foreign_keys = ON; - - CREATE TABLE IF NOT EXISTS sources ( - source_id TEXT PRIMARY KEY, - path TEXT NOT NULL, - kind TEXT NOT NULL, - mime TEXT NOT NULL, - sha256 TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - created_at TEXT NOT NULL, - metadata_json TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS atoms ( - atom_id TEXT PRIMARY KEY, - source_id TEXT NOT NULL REFERENCES sources(source_id) ON DELETE CASCADE, - modality TEXT NOT NULL, - source TEXT NOT NULL, - time TEXT NOT NULL, - text TEXT NOT NULL, - image_ref TEXT, - embedding_refs_json TEXT NOT NULL, - layout_json TEXT, - relations_json TEXT NOT NULL, - trust_json TEXT NOT NULL, - token_equivalent INTEGER NOT NULL, - metadata_json TEXT NOT NULL - ); - - CREATE INDEX IF NOT EXISTS idx_atoms_source_id ON atoms(source_id); - CREATE INDEX IF NOT EXISTS idx_atoms_modality ON atoms(modality); - CREATE VIRTUAL TABLE IF NOT EXISTS atom_fts USING fts5( - atom_id UNINDEXED, - source, - text, - tokenize = 'unicode61' - ); - - CREATE TABLE IF NOT EXISTS cartridges ( - cartridge_id TEXT PRIMARY KEY, - name TEXT NOT NULL, - root_path TEXT NOT NULL, - manifest_json TEXT NOT NULL, - dna TEXT, - token_equivalent INTEGER NOT NULL, - created_at TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS cache_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts TEXT NOT NULL, - layer TEXT NOT NULL, - cache_key TEXT NOT NULL, - event TEXT NOT NULL, - bytes_estimate INTEGER NOT NULL, - metadata_json TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS evidence_edges ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - from_atom_id TEXT NOT NULL, - to_atom_id TEXT NOT NULL, - relation TEXT NOT NULL, - confidence REAL NOT NULL, - metadata_json TEXT NOT NULL - ); - """ - ) + apply_migrations(connection) def upsert_sources(self, sources: list[SourceRecord]) -> None: with self.connect() as connection: @@ -150,15 +91,17 @@ class Database: connection.executemany( """ INSERT INTO atoms ( - atom_id, source_id, modality, source, time, text, image_ref, + atom_id, source_id, modality, source, ingested_at, text, image_ref, embedding_refs_json, layout_json, relations_json, trust_json, - token_equivalent, metadata_json + token_equivalent, metadata_json, holder_id, origin, scenario, + event_time, event_time_end, event_time_precision, event_time_source, + valid_from, valid_to, supersedes_json, visibility_json, retention_until ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(atom_id) DO UPDATE SET modality=excluded.modality, source=excluded.source, - time=excluded.time, + ingested_at=excluded.ingested_at, text=excluded.text, image_ref=excluded.image_ref, embedding_refs_json=excluded.embedding_refs_json, @@ -166,7 +109,19 @@ class Database: relations_json=excluded.relations_json, trust_json=excluded.trust_json, token_equivalent=excluded.token_equivalent, - metadata_json=excluded.metadata_json + metadata_json=excluded.metadata_json, + holder_id=excluded.holder_id, + origin=excluded.origin, + scenario=excluded.scenario, + event_time=excluded.event_time, + event_time_end=excluded.event_time_end, + event_time_precision=excluded.event_time_precision, + event_time_source=excluded.event_time_source, + valid_from=excluded.valid_from, + valid_to=excluded.valid_to, + supersedes_json=excluded.supersedes_json, + visibility_json=excluded.visibility_json, + retention_until=excluded.retention_until """, [self._atom_params(atom) for atom in atoms], ) @@ -182,7 +137,7 @@ class Database: atom.source_id, atom.modality.value, atom.source, - atom.time.isoformat(), + atom.ingested_at.isoformat(), atom.text, atom.image_ref, _json(atom.embedding_refs), @@ -191,25 +146,63 @@ class Database: _json(atom.trust.model_dump()), atom.token_equivalent, _json(atom.metadata), + atom.holder_id, + atom.origin.value, + atom.scenario, + atom.event_time.isoformat() if atom.event_time else None, + atom.event_time_end.isoformat() if atom.event_time_end else None, + atom.event_time_precision.value, + atom.event_time_source, + atom.valid_from.isoformat() if atom.valid_from else None, + atom.valid_to.isoformat() if atom.valid_to else None, + _json(atom.supersedes), + _json(atom.visibility.model_dump()), + atom.retention_until.isoformat() if atom.retention_until else None, ) def list_atoms(self, limit: int | None = None) -> list[EvidenceAtom]: - query = "SELECT * FROM atoms ORDER BY time DESC" - params: tuple[Any, ...] = () - if limit is not None: - query += " LIMIT ?" - params = (limit,) with self.connect() as connection: - rows = connection.execute(query, params).fetchall() + if limit is None: + rows = connection.execute("SELECT * FROM atoms ORDER BY ingested_at DESC").fetchall() + else: + rows = connection.execute( + "SELECT * FROM atoms ORDER BY ingested_at DESC LIMIT ?", (limit,) + ).fetchall() + return [self._row_to_atom(row) for row in rows] + + def atoms_as_of(self, moment: datetime, holder: str | None = None) -> list[EvidenceAtom]: + """Memories that were open at `moment`, the question a parameterless view cannot ask. + + Same predicate as v_atoms_current minus the retention filter: retention says how long + the system keeps a memory, not whether it was true then. Unknown bounds read as open. + """ + stamp = moment.isoformat() + with self.connect() as connection: + if holder is None: + rows = connection.execute( + "SELECT * FROM atoms " + "WHERE (valid_from IS NULL OR valid_from <= ?) " + "AND (valid_to IS NULL OR valid_to > ?) " + "ORDER BY ingested_at DESC", + (stamp, stamp), + ).fetchall() + else: + rows = connection.execute( + "SELECT * FROM atoms " + "WHERE holder_id = ? " + "AND (valid_from IS NULL OR valid_from <= ?) " + "AND (valid_to IS NULL OR valid_to > ?) " + "ORDER BY ingested_at DESC", + (holder, stamp, stamp), + ).fetchall() return [self._row_to_atom(row) for row in rows] def get_atoms(self, atom_ids: list[str]) -> list[EvidenceAtom]: if not atom_ids: return [] - placeholders = ",".join("?" for _ in atom_ids) with self.connect() as connection: rows = connection.execute( - f"SELECT * FROM atoms WHERE atom_id IN ({placeholders})", atom_ids + "SELECT * FROM atoms WHERE atom_id IN (SELECT value FROM json_each(?))", (_json(atom_ids),) ).fetchall() by_id = {row["atom_id"]: self._row_to_atom(row) for row in rows} return [by_id[atom_id] for atom_id in atom_ids if atom_id in by_id] @@ -318,7 +311,7 @@ class Database: VALUES (?, ?, ?, ?, ?, ?) """, ( - datetime.utcnow().isoformat(), + utc_now().isoformat(), layer, cache_key, event, @@ -349,16 +342,18 @@ class Database: ) def list_evidence_edges(self, limit: int | None = None) -> list[dict[str, Any]]: - query = ( - "SELECT from_atom_id, to_atom_id, relation, confidence, metadata_json " - "FROM evidence_edges ORDER BY id DESC" - ) - params: tuple[Any, ...] = () - if limit is not None: - query += " LIMIT ?" - params = (limit,) with self.connect() as connection: - rows = connection.execute(query, params).fetchall() + if limit is None: + rows = connection.execute( + "SELECT from_atom_id, to_atom_id, relation, confidence, metadata_json " + "FROM evidence_edges ORDER BY id DESC" + ).fetchall() + else: + rows = connection.execute( + "SELECT from_atom_id, to_atom_id, relation, confidence, metadata_json " + "FROM evidence_edges ORDER BY id DESC LIMIT ?", + (limit,), + ).fetchall() return [ { "from_atom_id": row["from_atom_id"], @@ -387,7 +382,7 @@ class Database: ).fetchall() def _row_to_atom(self, row: sqlite3.Row) -> EvidenceAtom: - from .schemas import LayoutBox, Modality, TrustScores + from .schemas import LayoutBox, Modality, TrustScores, Visibility layout_data = _loads(row["layout_json"], None) return EvidenceAtom( @@ -395,7 +390,7 @@ class Database: modality=Modality(row["modality"]), source_id=row["source_id"], source=row["source"], - time=datetime.fromisoformat(row["time"]), + ingested_at=datetime.fromisoformat(row["ingested_at"]), text=row["text"], image_ref=row["image_ref"], embedding_refs=_loads(row["embedding_refs_json"], []), @@ -404,6 +399,18 @@ class Database: trust=TrustScores(**_loads(row["trust_json"], {"parser": 0.0})), token_equivalent=int(row["token_equivalent"]), metadata=_loads(row["metadata_json"], {}), + holder_id=row["holder_id"], + origin=MemoryOrigin(row["origin"]), + scenario=row["scenario"], + event_time=_datetime(row["event_time"]), + event_time_end=_datetime(row["event_time_end"]), + event_time_precision=TimePrecision(row["event_time_precision"]), + event_time_source=row["event_time_source"], + valid_from=_datetime(row["valid_from"]), + valid_to=_datetime(row["valid_to"]), + supersedes=_loads(row["supersedes_json"], []), + visibility=Visibility(**_loads(row["visibility_json"], {})), + retention_until=_datetime(row["retention_until"]), ) def _row_to_source(self, row: sqlite3.Row) -> SourceRecord: diff --git a/backend/src/ely_eye/evals.py b/backend/src/ely_eye/evals.py index c40f2be..297a8cf 100644 --- a/backend/src/ely_eye/evals.py +++ b/backend/src/ely_eye/evals.py @@ -25,6 +25,11 @@ from .cartridge_assets import CartridgeAssetService, merkle_root from .compiler import ContextCompiler from .config import Settings, get_settings from .db import Database +from .evaluation_schemas import ( + HashHopProof, + ProofCheck, + ProofSuiteReport, +) from .evidence_graph import EvidenceGraphService from .ingestion import SUPPORTED_CODE_EXTENSIONS, build_repository_context, find_git_root from .profiles import runtime_profile_report @@ -36,10 +41,7 @@ from .schemas import ( CompiledContext, ContextPlan, EvidenceAtom, - HashHopProof, Modality, - ProofCheck, - ProofSuiteReport, RetrievalHit, RuntimeProfile, TrainingAdapterKind, @@ -56,7 +58,7 @@ from .training import ( sha256_file, sha256_tree, ) -from .verifier import VisualContradictionLens, Verifier +from .verifier import Verifier, VisualContradictionLens class HashHopEvaluator: @@ -101,11 +103,14 @@ class HashHopEvaluator: proof = self.generate_text_proof(hops, token_equivalent) root = Path(proof.artifacts[0]).parent chain_path = root / "hashhop_text.jsonl" - records = [ - json.loads(line) - for line in chain_path.read_text(encoding="utf-8").splitlines() - if line.strip() - ] + try: + chain_text = chain_path.read_text(encoding="utf-8") + except OSError as error: + raise ValueError(f"{chain_path}: the HashHop chain is unreadable ({error})") from error + try: + records = [json.loads(line) for line in chain_text.splitlines() if line.strip()] + except json.JSONDecodeError as error: + raise ValueError(f"{chain_path}: the HashHop chain holds a malformed line ({error})") from error decoys = [ {"from": secrets.token_hex(16), "to": secrets.token_hex(16)} for _ in range(max(4, hops * 2)) @@ -775,7 +780,7 @@ class PRDProofSuite: f"{citation_report.citation_accuracy:.2f}" ), ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run payload["error"] = str(exc) write_json(artifact, payload) return ProofCheck( @@ -876,7 +881,7 @@ class PRDProofSuite: "last_segment": segments[-1] if segments else None, } ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 # a probe records its failure instead of crashing the proof run payload["error"] = str(exc) write_json(artifact, payload) return ProofCheck( @@ -903,7 +908,10 @@ class PRDProofSuite: graph_path = cartridge_root / manifest.artifacts["temporal_graph"] computed_dna = self.cartridges.compute_dna(cartridge_root) atom_rows = parquet_row_count(atoms_path) - sources = json.loads(sources_path.read_text(encoding="utf-8")) if sources_path.exists() else [] + try: + sources = json.loads(sources_path.read_text(encoding="utf-8")) if sources_path.exists() else [] + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{sources_path}: the cartridge source list is unreadable ({error})") from error question = "Which Ely-Eye PRD requirements prove long context memory, HashHop, and adapters?" compiler = ContextCompiler(self.settings, self.db) replay_a = compiler.compile(question, RuntimeProfile.library_100m) @@ -1531,7 +1539,7 @@ def peft_config_loads(path: Path) -> bool: PeftConfig.from_pretrained(path) return True - except Exception: + except Exception: # noqa: BLE001 # a probe records its failure instead of crashing the proof run return False @@ -1585,7 +1593,7 @@ def training_payload_contract_ok(payload: dict[str, Any], manifest: AdapterManif and payload.get("precision") == TRAINING_PRECISION and payload.get("framework") == TRAINING_FRAMEWORK and payload.get("optimizer_family") == manifest.optimizer_family - and payload.get("gradient_checkpointing") is True + and bool(payload.get("gradient_checkpointing")) and payload.get("gradient_checkpointing_mode") == GRADIENT_CHECKPOINTING_MODE and payload.get("autocast_dtype") == AUTOCAST_DTYPE and payload.get("unsloth_version") == manifest.unsloth_version @@ -1876,10 +1884,13 @@ def read_memory_capsule(path: Path) -> dict[str, Any]: prefix = b"ELYMEMCAP1\n" if not data.startswith(prefix): raise ValueError(f"Invalid memory capsule header: {path}") - payload = zstd.ZstdDecompressor().decompress(data[len(prefix) :]) - parsed = json.loads(payload.decode("utf-8")) + try: + payload = zstd.ZstdDecompressor().decompress(data[len(prefix) :]) + parsed = json.loads(payload.decode("utf-8")) + except (zstd.ZstdError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"Cannot decode the memory capsule at {path}: {error}") from error if not isinstance(parsed, dict): - raise ValueError(f"Invalid memory capsule payload: {path}") + raise ValueError(f"Invalid memory capsule payload: {path}") # noqa: TRY004 # the capsule's own bytes are wrong, not the caller's argument return parsed @@ -1896,11 +1907,14 @@ def hashhop_chain_valid(proof: HashHopProof) -> bool: chain_artifacts = [Path(path) for path in proof.artifacts if path.endswith(".jsonl")] if len(chain_artifacts) != 1 or not chain_artifacts[0].exists(): return False - rows = [ - json.loads(line) - for line in chain_artifacts[0].read_text(encoding="utf-8").splitlines() - if line.strip() - ] + try: + rows = [ + json.loads(line) + for line in chain_artifacts[0].read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (OSError, json.JSONDecodeError): + return False # an unreadable chain is not a valid chain, and this is a predicate if len(rows) != proof.hops: return False if rows[0]["from"] != proof.query_id or rows[-1]["to"] != proof.expected_target_id: diff --git a/backend/src/ely_eye/evaluation_schemas.py b/backend/src/ely_eye/evaluation_schemas.py new file mode 100644 index 0000000..d2a624a --- /dev/null +++ b/backend/src/ely_eye/evaluation_schemas.py @@ -0,0 +1,122 @@ +"""Report shapes for the evaluation and proof layer. + +These describe what a run claims, not what the memory holds, so they live apart from the +memory and runtime schemas: a proof report changes when the evidence rules change, and the +memory model should not move with it. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, Field + +from .schemas import utc_now + + +class BenchmarkSpec(BaseModel): + name: str + target: str + dataset: str + task_type: str + runner: str + metric: str + +class BenchmarkRegistryReport(BaseModel): + status: Literal["ready", "incomplete"] + created_at: datetime = Field(default_factory=utc_now) + benchmarks: list[BenchmarkSpec] + +class BenchmarkSourceStatus(BaseModel): + name: str + dataset: str + status: Literal["ready", "unavailable"] + sha: str | None = None + gated: bool | str | None = None + private: bool | None = None + sibling_count: int = 0 + error: str | None = None + +class BenchmarkSourceReport(BaseModel): + status: Literal["ready", "incomplete"] + created_at: datetime = Field(default_factory=utc_now) + sources: list[BenchmarkSourceStatus] + +class BenchmarkSampleStatus(BaseModel): + name: str + dataset: str + source_dataset: str | None = None + source_sha: str | None = None + source_note: str | None = None + status: Literal["loaded", "gated", "failed"] + config: str | None = None + split: str | None = None + sample_id: str | None = None + fields: list[str] = Field(default_factory=list) + question_preview: str | None = None + answer_preview: str | None = None + sample_sha256: str | None = None + media_artifacts: list[str] = Field(default_factory=list) + error: str | None = None + +class BenchmarkSampleReport(BaseModel): + status: Literal["ready", "partial", "failed"] + created_at: datetime = Field(default_factory=utc_now) + samples: list[BenchmarkSampleStatus] + +class BenchmarkPredictionStatus(BaseModel): + name: str + dataset: str + status: Literal["predicted", "gated", "failed"] + config: str | None = None + split: str | None = None + sample_id: str | None = None + task_type: str + metric: str + prompt_sha256: str | None = None + sample_sha256: str | None = None + prediction: str | None = None + answer: str | None = None + score: float | None = None + runtime_backend: str | None = None + adapter_id: str | None = None + adapter_sha256: str | None = None + input_tokens: int | None = None + output_tokens: int | None = None + media_artifacts: list[str] = Field(default_factory=list) + error: str | None = None + +class BenchmarkPredictionReport(BaseModel): + status: Literal["ready", "partial", "failed"] + created_at: datetime = Field(default_factory=utc_now) + predictions: list[BenchmarkPredictionStatus] + +class HashHopProof(BaseModel): + proof_id: str + kind: Literal["text_hashhop", "visual_hashhop"] + hops: int + token_equivalent: int + query_id: str + expected_target_id: str + model_target_id: str | None = None + passed: bool | None = None + artifacts: list[str] + created_at: datetime = Field(default_factory=utc_now) + +class ProofCheck(BaseModel): + name: str + requirement: str + status: Literal["passed", "failed"] + evidence: list[str] + detail: str + +class ProofSuiteReport(BaseModel): + proof_id: str + cartridge_id: str + status: Literal["passed", "failed"] + created_at: datetime = Field(default_factory=utc_now) + checks: list[ProofCheck] + artifacts: dict[str, str] + dna_before: str | None = None + dna_after: str | None = None diff --git a/backend/src/ely_eye/migrations.py b/backend/src/ely_eye/migrations.py new file mode 100644 index 0000000..b4fd85e --- /dev/null +++ b/backend/src/ely_eye/migrations.py @@ -0,0 +1,183 @@ +"""Schema DDL and versioned migrations for the memory store. + +A memory row must answer four questions before retrieval can trust it: whose memory +is it, where did it come from, when was it true, and in what situation may it be used. +The first version of this table answered none of them, so every consumer guessed. + +Versioning uses PRAGMA user_version. A database written by the first version is +upgraded in place rather than rebuilt, because the only copy of the memory is the file. +""" + +from __future__ import annotations + +import sqlite3 + +SCHEMA_VERSION = 2 + +ATOM_COLUMNS_V2 = """ + atom_id TEXT PRIMARY KEY, + source_id TEXT NOT NULL REFERENCES sources(source_id) ON DELETE CASCADE, + modality TEXT NOT NULL, + source TEXT NOT NULL, + ingested_at TEXT NOT NULL, + text TEXT NOT NULL, + image_ref TEXT, + embedding_refs_json TEXT NOT NULL, + layout_json TEXT, + relations_json TEXT NOT NULL, + trust_json TEXT NOT NULL, + token_equivalent INTEGER NOT NULL, + metadata_json TEXT NOT NULL, + holder_id TEXT NOT NULL DEFAULT '', + origin TEXT NOT NULL DEFAULT 'artifact' + CHECK (origin IN ('self', 'observed', 'artifact', 'third_party')), + scenario TEXT NOT NULL DEFAULT '', + event_time TEXT, + event_time_end TEXT, + event_time_precision TEXT NOT NULL DEFAULT 'unknown' + CHECK (event_time_precision IN ('instant', 'minute', 'hour', 'day', 'month', 'year', 'unknown')), + event_time_source TEXT NOT NULL DEFAULT '', + valid_from TEXT, + valid_to TEXT, + supersedes_json TEXT NOT NULL DEFAULT '[]', + visibility_json TEXT NOT NULL DEFAULT '{}', + retention_until TEXT +""" + +_BASE_TABLES = """ + CREATE TABLE IF NOT EXISTS sources ( + source_id TEXT PRIMARY KEY, + path TEXT NOT NULL, + kind TEXT NOT NULL, + mime TEXT NOT NULL, + sha256 TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + created_at TEXT NOT NULL, + metadata_json TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS cartridges ( + cartridge_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + root_path TEXT NOT NULL, + manifest_json TEXT NOT NULL, + dna TEXT, + token_equivalent INTEGER NOT NULL, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS cache_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + layer TEXT NOT NULL, + cache_key TEXT NOT NULL, + event TEXT NOT NULL, + bytes_estimate INTEGER NOT NULL, + metadata_json TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS evidence_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + from_atom_id TEXT NOT NULL, + to_atom_id TEXT NOT NULL, + relation TEXT NOT NULL, + confidence REAL NOT NULL, + metadata_json TEXT NOT NULL + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS atom_fts USING fts5( + atom_id UNINDEXED, + source, + text, + tokenize = 'unicode61' + ); +""" + +# A memory is usable now when nothing has closed it, it had begun by now, and its +# retention has not lapsed. Unknown bounds read as visible: absence of evidence about +# when a legacy row was true is not evidence that it is false. The clock is formatted to +# match how Python writes an aware datetime, so the comparison is exact to the millisecond. +VIEWS = """ + CREATE VIEW IF NOT EXISTS v_atoms_current AS + SELECT * FROM atoms + WHERE valid_to IS NULL + AND (valid_from IS NULL OR valid_from <= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now')) + AND (retention_until IS NULL OR retention_until >= strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now')); + + CREATE INDEX IF NOT EXISTS idx_atoms_source_id ON atoms(source_id); + CREATE INDEX IF NOT EXISTS idx_atoms_modality ON atoms(modality); + CREATE INDEX IF NOT EXISTS idx_atoms_holder ON atoms(holder_id); + CREATE INDEX IF NOT EXISTS idx_atoms_scenario ON atoms(scenario); + CREATE INDEX IF NOT EXISTS idx_atoms_valid_to ON atoms(valid_to); +""" + +# Columns the first version lacked, in the order they are added. Every one is nullable +# or has a default, so an existing row survives the upgrade with its meaning intact. +_V1_TO_V2_COLUMNS = ( + ("holder_id", "TEXT NOT NULL DEFAULT ''"), + ("origin", "TEXT NOT NULL DEFAULT 'artifact'"), + ("scenario", "TEXT NOT NULL DEFAULT ''"), + ("event_time", "TEXT"), + ("event_time_end", "TEXT"), + ("event_time_precision", "TEXT NOT NULL DEFAULT 'unknown'"), + ("event_time_source", "TEXT NOT NULL DEFAULT ''"), + ("valid_from", "TEXT"), + ("valid_to", "TEXT"), + ("supersedes_json", "TEXT NOT NULL DEFAULT '[]'"), + ("visibility_json", "TEXT NOT NULL DEFAULT '{}'"), + ("retention_until", "TEXT"), +) + + +def _table_exists(connection: sqlite3.Connection, name: str) -> bool: + row = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,) + ).fetchone() + return row is not None + + +def _column_names(connection: sqlite3.Connection, table: str) -> set[str]: + return {str(row[1]) for row in connection.execute(f"PRAGMA table_info({table})")} + + +def _create_v2(connection: sqlite3.Connection) -> None: + connection.executescript(_BASE_TABLES) + connection.execute(f"CREATE TABLE IF NOT EXISTS atoms ({ATOM_COLUMNS_V2})") + + +def _upgrade_v1_to_v2(connection: sqlite3.Connection) -> None: + """Add the four memory dimensions to a first-version atoms table. + + The old `time` column recorded when ingestion ran, which is the only timestamp a + legacy row honestly has, so it is renamed rather than duplicated. + """ + columns = _column_names(connection, "atoms") + if "time" in columns and "ingested_at" not in columns: + connection.execute("ALTER TABLE atoms RENAME COLUMN time TO ingested_at") + columns = _column_names(connection, "atoms") + if "ingested_at" not in columns: + raise SystemExit( + "atoms table has neither `time` nor `ingested_at`; refusing to guess which timestamp is ingestion" + ) + for name, definition in _V1_TO_V2_COLUMNS: + if name not in columns: + connection.execute(f"ALTER TABLE atoms ADD COLUMN {name} {definition}") + + +def apply(connection: sqlite3.Connection) -> None: + """Bring the schema to SCHEMA_VERSION, in place, without losing rows.""" + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + if version > SCHEMA_VERSION: + raise SystemExit(f"database schema is version {version}, this build understands {SCHEMA_VERSION}") + if version == SCHEMA_VERSION: + return + + if _table_exists(connection, "atoms"): + connection.executescript(_BASE_TABLES) + if version < 2: + _upgrade_v1_to_v2(connection) + else: + _create_v2(connection) + + connection.executescript(VIEWS) + connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") diff --git a/backend/src/ely_eye/schemas.py b/backend/src/ely_eye/schemas.py index c7c7919..8150a79 100644 --- a/backend/src/ely_eye/schemas.py +++ b/backend/src/ely_eye/schemas.py @@ -1,16 +1,15 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from enum import StrEnum from pathlib import Path from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator def utc_now() -> datetime: - return datetime.now(timezone.utc) - + return datetime.now(UTC) class RuntimeProfile(StrEnum): live_demo = "live_demo" @@ -18,7 +17,6 @@ class RuntimeProfile(StrEnum): library_100m = "library_100m" research_theater = "research_theater" - class TrainingAdapterKind(StrEnum): hme_core_lora = "hme_core_lora" hme_vision_lora = "hme_vision_lora" @@ -27,7 +25,6 @@ class TrainingAdapterKind(StrEnum): hme_router = "hme_router" hme_retrieval = "hme_retrieval" - class Modality(StrEnum): text = "text" code = "code" @@ -38,6 +35,36 @@ class Modality(StrEnum): ui_screenshot = "ui_screenshot" mixed = "mixed" +class MemoryOrigin(StrEnum): + """Where a memory came from, which decides how much it may be trusted. + + A file the user handed over is not the same kind of claim as something the system + watched happen, and neither is a statement the user made about themselves. + """ + + self_ = "self" + observed = "observed" + artifact = "artifact" + third_party = "third_party" + +class TimePrecision(StrEnum): + """How finely the event time is actually known, so a day is never read as an instant.""" + + instant = "instant" + minute = "minute" + hour = "hour" + day = "day" + month = "month" + year = "year" + unknown = "unknown" + +class Visibility(BaseModel): + """Who may use a memory. Retention lives on the atom itself because the current + memory view has to filter on it.""" + + scope: Literal["holder", "visible_to", "everyone"] = "holder" + holders: list[str] = Field(default_factory=list) + purpose: str | None = None class LayoutBox(BaseModel): x: float @@ -47,19 +74,17 @@ class LayoutBox(BaseModel): page: int | None = None frame_second: float | None = None - class TrustScores(BaseModel): parser: float = Field(ge=0.0, le=1.0) ocr: float = Field(default=0.0, ge=0.0, le=1.0) model: float = Field(default=0.0, ge=0.0, le=1.0) - class EvidenceAtom(BaseModel): atom_id: str modality: Modality source_id: str source: str - time: datetime = Field(default_factory=utc_now) + ingested_at: datetime = Field(default_factory=utc_now) text: str = "" image_ref: str | None = None embedding_refs: list[str] = Field(default_factory=list) @@ -68,7 +93,28 @@ class EvidenceAtom(BaseModel): trust: TrustScores token_equivalent: int = Field(default=0, ge=0) metadata: dict[str, Any] = Field(default_factory=dict) + holder_id: str = "" + origin: MemoryOrigin = MemoryOrigin.artifact + scenario: str = "" + event_time: datetime | None = None + event_time_end: datetime | None = None + event_time_precision: TimePrecision = TimePrecision.unknown + event_time_source: str = "" + valid_from: datetime | None = None + valid_to: datetime | None = None + supersedes: list[str] = Field(default_factory=list) + visibility: Visibility = Field(default_factory=Visibility) + retention_until: datetime | None = None + @model_validator(mode="after") + def _times_are_ordered(self) -> EvidenceAtom: + if self.event_time and self.event_time_end and self.event_time_end < self.event_time: + raise ValueError(f"{self.atom_id}: event_time_end precedes event_time") + if self.valid_from and self.valid_to and self.valid_to < self.valid_from: + raise ValueError(f"{self.atom_id}: valid_to precedes valid_from") + if self.atom_id in self.supersedes: + raise ValueError(f"{self.atom_id}: an atom cannot supersede itself") + return self class SourceRecord(BaseModel): source_id: str @@ -80,7 +126,6 @@ class SourceRecord(BaseModel): created_at: datetime = Field(default_factory=utc_now) metadata: dict[str, Any] = Field(default_factory=dict) - class IngestResponse(BaseModel): source_count: int atom_count: int @@ -88,7 +133,6 @@ class IngestResponse(BaseModel): token_equivalent: int sources: list[SourceRecord] - class RetrievalHit(BaseModel): atom: EvidenceAtom sparse_score: float @@ -96,7 +140,6 @@ class RetrievalHit(BaseModel): graph_score: float = 0.0 final_score: float - class ContextPlan(BaseModel): question: str profile: RuntimeProfile @@ -107,7 +150,6 @@ class ContextPlan(BaseModel): compression_strategy: str verifier_contract: list[str] - class CompiledContext(BaseModel): plan: ContextPlan hits: list[RetrievalHit] @@ -115,7 +157,6 @@ class CompiledContext(BaseModel): token_equivalent: int cache_trace_id: str - class CitationReport(BaseModel): cited_atom_ids: list[str] missing_atom_ids: list[str] @@ -123,7 +164,6 @@ class CitationReport(BaseModel): contradiction_notes: list[str] confidence: float - class AnswerResponse(BaseModel): answer: str citations: list[str] @@ -131,7 +171,6 @@ class AnswerResponse(BaseModel): verifier: CitationReport runtime_backend: str - class CacheLayerStatus(BaseModel): layer: str entries: int @@ -140,7 +179,6 @@ class CacheLayerStatus(BaseModel): miss_count: int last_event_at: datetime | None - class MemoryStatus(BaseModel): active_atoms: int active_token_equivalent: int @@ -148,7 +186,6 @@ class MemoryStatus(BaseModel): library_token_equivalent: int layers: list[CacheLayerStatus] - class RuntimeStatus(BaseModel): backend: str model_id: str @@ -163,7 +200,6 @@ class RuntimeStatus(BaseModel): vram_total_mb: int | None = None vram_used_mb: int | None = None - class SystemStatus(BaseModel): app: str = "Ely-Eye" version: str @@ -172,7 +208,6 @@ class SystemStatus(BaseModel): profiles: dict[str, dict[str, Any]] data_home: str - class CartridgeManifest(BaseModel): cartridge_id: str name: str @@ -184,7 +219,6 @@ class CartridgeManifest(BaseModel): artifacts: dict[str, str] dna: str | None = None - class AdapterManifest(BaseModel): adapter_id: str kind: TrainingAdapterKind @@ -248,7 +282,6 @@ class AdapterManifest(BaseModel): sha256: str cartridge_id: str | None = None - class AdapterStatus(BaseModel): name: str kind: TrainingAdapterKind @@ -299,7 +332,6 @@ class AdapterStatus(BaseModel): adapter_max_abs: float | None = None adapter_weights_finite: bool = False - class CartridgeAssetStatus(BaseModel): name: str path: str @@ -309,99 +341,12 @@ class CartridgeAssetStatus(BaseModel): dtype: str status: Literal["ready", "missing"] - class CartridgeAssetReport(BaseModel): cartridge_id: str status: Literal["ready", "incomplete"] created_at: datetime = Field(default_factory=utc_now) assets: list[CartridgeAssetStatus] - -class BenchmarkSpec(BaseModel): - name: str - target: str - dataset: str - task_type: str - runner: str - metric: str - - -class BenchmarkRegistryReport(BaseModel): - status: Literal["ready", "incomplete"] - created_at: datetime = Field(default_factory=utc_now) - benchmarks: list[BenchmarkSpec] - - -class BenchmarkSourceStatus(BaseModel): - name: str - dataset: str - status: Literal["ready", "unavailable"] - sha: str | None = None - gated: bool | str | None = None - private: bool | None = None - sibling_count: int = 0 - error: str | None = None - - -class BenchmarkSourceReport(BaseModel): - status: Literal["ready", "incomplete"] - created_at: datetime = Field(default_factory=utc_now) - sources: list[BenchmarkSourceStatus] - - -class BenchmarkSampleStatus(BaseModel): - name: str - dataset: str - source_dataset: str | None = None - source_sha: str | None = None - source_note: str | None = None - status: Literal["loaded", "gated", "failed"] - config: str | None = None - split: str | None = None - sample_id: str | None = None - fields: list[str] = Field(default_factory=list) - question_preview: str | None = None - answer_preview: str | None = None - sample_sha256: str | None = None - media_artifacts: list[str] = Field(default_factory=list) - error: str | None = None - - -class BenchmarkSampleReport(BaseModel): - status: Literal["ready", "partial", "failed"] - created_at: datetime = Field(default_factory=utc_now) - samples: list[BenchmarkSampleStatus] - - -class BenchmarkPredictionStatus(BaseModel): - name: str - dataset: str - status: Literal["predicted", "gated", "failed"] - config: str | None = None - split: str | None = None - sample_id: str | None = None - task_type: str - metric: str - prompt_sha256: str | None = None - sample_sha256: str | None = None - prediction: str | None = None - answer: str | None = None - score: float | None = None - runtime_backend: str | None = None - adapter_id: str | None = None - adapter_sha256: str | None = None - input_tokens: int | None = None - output_tokens: int | None = None - media_artifacts: list[str] = Field(default_factory=list) - error: str | None = None - - -class BenchmarkPredictionReport(BaseModel): - status: Literal["ready", "partial", "failed"] - created_at: datetime = Field(default_factory=utc_now) - predictions: list[BenchmarkPredictionStatus] - - class RuntimeLaunchProfile(BaseModel): name: str runtime: str @@ -412,7 +357,6 @@ class RuntimeLaunchProfile(BaseModel): environment: dict[str, str] = Field(default_factory=dict) required_features: list[str] = Field(default_factory=list) - class RuntimeProfileValidation(BaseModel): name: str runtime: str @@ -428,51 +372,16 @@ class RuntimeProfileValidation(BaseModel): probe_output: str | None = None detail: str - class RuntimeProfileReport(BaseModel): status: Literal["ready", "incomplete"] created_at: datetime = Field(default_factory=utc_now) profiles: list[RuntimeLaunchProfile] validations: list[RuntimeProfileValidation] = Field(default_factory=list) - -class HashHopProof(BaseModel): - proof_id: str - kind: Literal["text_hashhop", "visual_hashhop"] - hops: int - token_equivalent: int - query_id: str - expected_target_id: str - model_target_id: str | None = None - passed: bool | None = None - artifacts: list[str] - created_at: datetime = Field(default_factory=utc_now) - - -class ProofCheck(BaseModel): - name: str - requirement: str - status: Literal["passed", "failed"] - evidence: list[str] - detail: str - - -class ProofSuiteReport(BaseModel): - proof_id: str - cartridge_id: str - status: Literal["passed", "failed"] - created_at: datetime = Field(default_factory=utc_now) - checks: list[ProofCheck] - artifacts: dict[str, str] - dna_before: str | None = None - dna_after: str | None = None - - class UploadIngestRequest(BaseModel): cartridge_name: str | None = None dense_embeddings: bool | None = None - class PathIngestRequest(BaseModel): path: Path cartridge_name: str | None = None diff --git a/backend/tests/test_memory_schema.py b/backend/tests/test_memory_schema.py new file mode 100644 index 0000000..1960e88 --- /dev/null +++ b/backend/tests/test_memory_schema.py @@ -0,0 +1,224 @@ +"""The memory model must carry holder, origin, time and scenario, and an old database must survive.""" + +from __future__ import annotations + +import sqlite3 +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from ely_eye.config import Settings +from ely_eye.db import Database +from ely_eye.migrations import SCHEMA_VERSION +from ely_eye.schemas import ( + EvidenceAtom, + MemoryOrigin, + Modality, + SourceRecord, + TimePrecision, + TrustScores, + Visibility, +) + +NOW = datetime(2026, 5, 1, 12, 0, tzinfo=UTC) + + +def _atom(atom_id: str, **overrides: object) -> EvidenceAtom: + fields: dict[str, object] = { + "atom_id": atom_id, + "modality": Modality.text, + "source_id": "src-1", + "source": "notes/key.txt", + "ingested_at": NOW, + "text": "钥匙放在玄关抽屉里", + "trust": TrustScores(parser=1.0), + "token_equivalent": 8, + "holder_id": "wife", + "origin": MemoryOrigin.self_, + "scenario": "home", + "event_time": NOW - timedelta(days=1), + "event_time_precision": TimePrecision.day, + "event_time_source": "declared", + "valid_from": NOW - timedelta(days=1), + } + fields.update(overrides) + return EvidenceAtom(**fields) # type: ignore[arg-type] + + +def _database(home: Path) -> Database: + return Database(Settings(home=home)) + + +def _seed_source(database: Database) -> None: + database.upsert_sources( + [ + SourceRecord( + source_id="src-1", + path="/tmp/key.txt", + kind="text", + mime="text/plain", + sha256="0" * 64, + size_bytes=12, + created_at=NOW, + ) + ] + ) + + +def _atom_columns(database: Database) -> set[str]: + with database.connect() as connection: + return {row[1] for row in connection.execute("PRAGMA table_info(atoms)")} + + +def test_a_fresh_database_carries_the_four_memory_dimensions(tmp_path: Path) -> None: + database = _database(tmp_path) + columns = _atom_columns(database) + for column in ( + "holder_id", + "origin", + "scenario", + "event_time", + "event_time_end", + "event_time_precision", + "event_time_source", + "valid_from", + "valid_to", + "supersedes_json", + "visibility_json", + "retention_until", + "ingested_at", + ): + assert column in columns, column + assert "time" not in columns, "ingestion time has one name: ingested_at" + with database.connect() as connection: + assert int(connection.execute("PRAGMA user_version").fetchone()[0]) == SCHEMA_VERSION + + +def test_an_atom_round_trips_every_memory_dimension(tmp_path: Path) -> None: + database = _database(tmp_path) + _seed_source(database) + atom = _atom( + "atom-1", + event_time_end=NOW - timedelta(hours=6), + valid_to=NOW + timedelta(days=2), + supersedes=["atom-0"], + visibility=Visibility(scope="visible_to", holders=["husband"], purpose="household"), + retention_until=NOW + timedelta(days=30), + ) + database.upsert_atoms([atom]) + stored = database.get_atoms(["atom-1"])[0] + assert stored.holder_id == "wife" + assert stored.origin is MemoryOrigin.self_ + assert stored.scenario == "home" + assert stored.event_time == atom.event_time + assert stored.event_time_end == atom.event_time_end + assert stored.event_time_precision is TimePrecision.day + assert stored.event_time_source == "declared" + assert stored.valid_from == atom.valid_from + assert stored.valid_to == atom.valid_to + assert stored.supersedes == ["atom-0"] + assert stored.visibility.holders == ["husband"] + assert stored.visibility.purpose == "household" + assert stored.retention_until == atom.retention_until + assert stored.ingested_at == NOW + + +def test_a_first_version_database_upgrades_in_place_and_keeps_its_rows(tmp_path: Path) -> None: + path = tmp_path / "data" / "ely_eye.sqlite" + path.parent.mkdir(parents=True) + connection = sqlite3.connect(path) + connection.executescript( + """ + CREATE TABLE sources ( + source_id TEXT PRIMARY KEY, path TEXT NOT NULL, kind TEXT NOT NULL, mime TEXT NOT NULL, + sha256 TEXT NOT NULL, size_bytes INTEGER NOT NULL, created_at TEXT NOT NULL, + metadata_json TEXT NOT NULL + ); + CREATE TABLE atoms ( + atom_id TEXT PRIMARY KEY, source_id TEXT NOT NULL, modality TEXT NOT NULL, + source TEXT NOT NULL, time TEXT NOT NULL, text TEXT NOT NULL, image_ref TEXT, + embedding_refs_json TEXT NOT NULL, layout_json TEXT, relations_json TEXT NOT NULL, + trust_json TEXT NOT NULL, token_equivalent INTEGER NOT NULL, metadata_json TEXT NOT NULL + ); + INSERT INTO atoms VALUES ( + 'legacy-1', 'src-1', 'text', 'old.txt', '2026-01-02T03:04:05+00:00', 'old memory', + NULL, '[]', NULL, '[]', '{"parser": 1.0}', 4, '{}' + ); + """ + ) + connection.commit() + connection.close() + + database = _database(tmp_path) + assert _atom_columns(database) >= {"holder_id", "origin", "valid_from", "ingested_at"} + legacy = database.get_atoms(["legacy-1"])[0] + assert legacy.text == "old memory" + assert legacy.ingested_at == datetime.fromisoformat("2026-01-02T03:04:05+00:00") + assert legacy.holder_id == "", "a legacy row has no known holder, and must not invent one" + assert legacy.origin is MemoryOrigin.artifact + + +def test_v_atoms_current_hides_closed_retired_and_unbegun_memories(tmp_path: Path) -> None: + """The view answers 'what is usable right now', so it is written against the real clock.""" + moment = datetime.now(UTC) + database = _database(tmp_path) + _seed_source(database) + database.upsert_atoms( + [ + _atom("live"), + _atom("closed", valid_to=moment - timedelta(days=1)), + _atom("lapsed", retention_until=moment - timedelta(days=1)), + _atom("future", valid_from=moment + timedelta(days=5)), + ] + ) + with database.connect() as connection: + visible = {row["atom_id"] for row in connection.execute("SELECT atom_id FROM v_atoms_current")} + assert visible == {"live"} + assert database.atom_count() == 4, "the view hides, it does not delete" + + +def test_an_unknown_origin_is_refused_by_the_database(tmp_path: Path) -> None: + database = _database(tmp_path) + _seed_source(database) + with database.connect() as connection, pytest.raises(sqlite3.IntegrityError): + connection.execute( + "INSERT INTO atoms (atom_id, source_id, modality, source, ingested_at, text, " + "embedding_refs_json, relations_json, trust_json, token_equivalent, metadata_json, origin) " + "VALUES ('x', 'src-1', 'text', 's', '2026-05-01T12:00:00+00:00', 't', '[]', '[]', " + "'{\"parser\": 1.0}', 1, '{}', 'whispered')" + ) + + +def test_the_schema_refuses_a_database_newer_than_this_build(tmp_path: Path) -> None: + database = _database(tmp_path) + with database.connect() as connection: + connection.execute("PRAGMA user_version = 99") + with pytest.raises(SystemExit, match="understands"): + _database(tmp_path) + + +def test_times_must_be_ordered_and_an_atom_cannot_supersede_itself() -> None: + with pytest.raises(ValueError, match="valid_to precedes valid_from"): + _atom("bad-1", valid_from=NOW, valid_to=NOW - timedelta(days=1)) + with pytest.raises(ValueError, match="event_time_end precedes event_time"): + _atom("bad-2", event_time=NOW, event_time_end=NOW - timedelta(days=1)) + with pytest.raises(ValueError, match="cannot supersede itself"): + _atom("bad-3", supersedes=["bad-3"]) + + +def test_atoms_as_of_walks_the_validity_timeline(tmp_path: Path) -> None: + """Time travel is a method because a SQLite view cannot take a parameter.""" + database = _database(tmp_path) + _seed_source(database) + database.upsert_atoms( + [ + _atom("live"), + _atom("closed", valid_from=NOW - timedelta(days=10), valid_to=NOW - timedelta(days=1)), + _atom("future", valid_from=NOW + timedelta(days=5)), + ] + ) + assert {atom.atom_id for atom in database.atoms_as_of(NOW)} == {"live"} + assert {atom.atom_id for atom in database.atoms_as_of(NOW - timedelta(days=2))} == {"closed"} + assert database.atoms_as_of(NOW, holder="nobody") == [] + assert {atom.atom_id for atom in database.atoms_as_of(NOW, holder="wife")} == {"live"} diff --git a/bench/reports/BASELINE_EXTERNAL.md b/bench/reports/BASELINE_EXTERNAL.md new file mode 100644 index 0000000..61ca11c --- /dev/null +++ b/bench/reports/BASELINE_EXTERNAL.md @@ -0,0 +1,64 @@ +# BASELINE_EXTERNAL: memory counterparts on the same corpus + +- generated 2026-10-07T17:26:07+00:00 at commit c4e2982506fa +- 5308 queries (2928 answerable) against 50789 sources +- machine: 32 cores, load average 1.5 when the report was written +- extraction is lexical for both stores; Mem0's real add() and Zep's graph builder call a generator model, which this round does not have + +| store | items | closed | recall@10 | hit@10 | hit@10 Wilson 95% lower | MRR | abstain (answerable) | abstain (unanswerable) | p50 s | p95 s | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| mem0_flat [holder scope] | 403859 | 0 | 0.866 | 0.867 | 0.855 | 0.629 | 0.000 | 0.000 | 0.084 | 0.133 | +| mem0_flat [whole corpus] | 403859 | 0 | 0.412 | 0.416 | 0.398 | 0.283 | 0.000 | 0.000 | 0.090 | 0.174 | +| zep_temporal [holder scope] | 403859 | 27072 | 0.866 | 0.867 | 0.854 | 0.627 | 0.000 | 0.058 | 0.088 | 0.111 | +| zep_temporal [whole corpus] | 403859 | 27072 | 0.429 | 0.433 | 0.415 | 0.297 | 0.000 | 0.000 | 0.086 | 0.107 | + +n per modality: {"code": 2474, "conversation": 300, "image": 480, "pdf": 984, "ui_screenshot": 640, "video": 430} + + +`[holder scope]` answers only from the asking holder's memory; `[whole corpus]` searches every +holder's memory, which is what the product's arms are shown. The gap between the two rows is the +value of the holder boundary, not of a better ranker. + +## mem0_flat [holder scope] by modality + +| modality | n answerable | recall@10 | hit@10 | Wilson lower | MRR | +| --- | --- | --- | --- | --- | --- | +| code | 1237 | 0.952 | 0.952 | 0.939 | 0.695 | +| conversation | 150 | 0.993 | 1.000 | 0.975 | 0.927 | +| image | 240 | 0.646 | 0.654 | 0.592 | 0.505 | +| pdf | 766 | 0.916 | 0.916 | 0.895 | 0.692 | +| ui_screenshot | 320 | 0.622 | 0.622 | 0.568 | 0.330 | +| video | 215 | 0.716 | 0.716 | 0.653 | 0.396 | + +## mem0_flat [whole corpus] by modality + +| modality | n answerable | recall@10 | hit@10 | Wilson lower | MRR | +| --- | --- | --- | --- | --- | --- | +| code | 1237 | 0.349 | 0.349 | 0.323 | 0.216 | +| conversation | 150 | 0.500 | 0.573 | 0.493 | 0.373 | +| image | 240 | 0.525 | 0.533 | 0.470 | 0.362 | +| pdf | 766 | 0.683 | 0.683 | 0.649 | 0.515 | +| ui_screenshot | 320 | 0.075 | 0.075 | 0.051 | 0.033 | +| video | 215 | 0.116 | 0.116 | 0.080 | 0.069 | + +## zep_temporal [holder scope] by modality + +| modality | n answerable | recall@10 | hit@10 | Wilson lower | MRR | +| --- | --- | --- | --- | --- | --- | +| code | 1237 | 0.952 | 0.952 | 0.939 | 0.695 | +| conversation | 150 | 0.978 | 0.987 | 0.953 | 0.905 | +| image | 240 | 0.646 | 0.654 | 0.592 | 0.505 | +| pdf | 766 | 0.916 | 0.916 | 0.895 | 0.692 | +| ui_screenshot | 320 | 0.622 | 0.622 | 0.568 | 0.330 | +| video | 215 | 0.716 | 0.716 | 0.653 | 0.396 | + +## zep_temporal [whole corpus] by modality + +| modality | n answerable | recall@10 | hit@10 | Wilson lower | MRR | +| --- | --- | --- | --- | --- | --- | +| code | 1237 | 0.350 | 0.350 | 0.324 | 0.216 | +| conversation | 150 | 0.591 | 0.667 | 0.588 | 0.429 | +| image | 240 | 0.562 | 0.571 | 0.508 | 0.426 | +| pdf | 766 | 0.685 | 0.685 | 0.652 | 0.519 | +| ui_screenshot | 320 | 0.119 | 0.119 | 0.088 | 0.057 | +| video | 215 | 0.163 | 0.163 | 0.119 | 0.087 | diff --git a/bench/reports/baseline_external.json b/bench/reports/baseline_external.json index 9ffc069..c8b02b8 100644 --- a/bench/reports/baseline_external.json +++ b/bench/reports/baseline_external.json @@ -2,15 +2,15 @@ "meta": { "answerable": 2928, "extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator", - "generated_at": "2026-10-07T16:54:23+00:00", + "generated_at": "2026-10-07T17:26:07+00:00", "git_commit": "c4e2982506faaf350a4b06738923899fc44bdedd", "limit_per_modality": null, "machine": { "cpus": 32, "load_average_1_5_15": [ - 6.23, - 4.72, - 4.44 + 1.47, + 2.81, + 10.33 ], "release": "6.18.40.1-microsoft-standard-WSL2", "system": "Linux" @@ -42,7 +42,7 @@ "recall_at_1": 0.549717057396928, "recall_at_10": 0.952303961196443, "recall_at_5": 0.8884397736459175, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "conversation": { "abstention_rate_answerable": 0.0, @@ -56,7 +56,7 @@ "recall_at_1": 0.657888888888889, "recall_at_10": 0.9927777777777778, "recall_at_5": 0.9814444444444445, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "image": { "abstention_rate_answerable": 0.0, @@ -70,7 +70,7 @@ "recall_at_1": 0.42291666666666666, "recall_at_10": 0.6458333333333334, "recall_at_5": 0.6, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "pdf": { "abstention_rate_answerable": 0.0, @@ -84,7 +84,7 @@ "recall_at_1": 0.587467362924282, "recall_at_10": 0.9164490861618799, "recall_at_5": 0.825065274151436, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, @@ -98,7 +98,7 @@ "recall_at_1": 0.21875, "recall_at_10": 0.621875, "recall_at_5": 0.465625, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "video": { "abstention_rate_answerable": 0.0, @@ -112,16 +112,16 @@ "recall_at_1": 0.2837209302325581, "recall_at_10": 0.7162790697674418, "recall_at_5": 0.5255813953488372, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 } }, "closed_items": 0, "items": 403859, "latency_seconds": { - "max": 15.670968882004672, - "mean": 0.09444417075096513, - "p50": 0.08692938649983262, - "p95": 0.10732038790265505, + "max": 7.547149860001809, + "mean": 0.09187759584445908, + "p50": 0.08432659949903609, + "p95": 0.13331470334669573, "queries": 5308 }, "overall": { @@ -136,9 +136,121 @@ "recall_at_1": 0.49903802367941713, "recall_at_10": 0.8664332877959927, "recall_at_5": 0.7801286429872496, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, - "store": "mem0_flat" + "store": "mem0_flat [holder scope]" + }, + { + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.349232012934519, + "hit_at_10_wilson_lower": 0.3231693767503265, + "mrr": 0.21579377911229164, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.24771932445387607, + "recall_at_1": 0.15683104284559418, + "recall_at_10": 0.349232012934519, + "recall_at_5": 0.2991107518189167, + "top_hit_from_holder_unanswerable": 0.0 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5733333333333334, + "hit_at_10_wilson_lower": 0.4933252687249978, + "mrr": 0.37327513227513226, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.38064109675098423, + "recall_at_1": 0.2188888888888889, + "recall_at_10": 0.5004444444444445, + "recall_at_5": 0.4146666666666667, + "top_hit_from_holder_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5333333333333333, + "hit_at_10_wilson_lower": 0.4701883065323697, + "mrr": 0.36166170634920636, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.3979762690192957, + "recall_at_1": 0.26458333333333334, + "recall_at_10": 0.525, + "recall_at_5": 0.4708333333333333, + "top_hit_from_holder_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6827676240208878, + "hit_at_10_wilson_lower": 0.6489674562308053, + "mrr": 0.5150985328857391, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.555124937448977, + "recall_at_1": 0.43994778067885115, + "recall_at_10": 0.6827676240208878, + "recall_at_5": 0.6109660574412533, + "top_hit_from_holder_unanswerable": 0.0 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.075, + "hit_at_10_wilson_lower": 0.05091491480619657, + "mrr": 0.032880704365079366, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.04277852579814079, + "recall_at_1": 0.01875, + "recall_at_10": 0.075, + "recall_at_5": 0.059375, + "top_hit_from_holder_unanswerable": 0.0 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.11627906976744186, + "hit_at_10_wilson_lower": 0.08001303523895324, + "mrr": 0.06852713178294573, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.08038135054972745, + "recall_at_1": 0.04186046511627907, + "recall_at_10": 0.11627906976744186, + "recall_at_5": 0.11627906976744186, + "top_hit_from_holder_unanswerable": 0.0 + } + }, + "closed_items": 0, + "items": 403859, + "latency_seconds": { + "max": 2.318251260003308, + "mean": 0.10140667538014644, + "p50": 0.09026374550012406, + "p95": 0.17443375029943126, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.41598360655737704, + "hit_at_10_wilson_lower": 0.39825196843019733, + "mrr": 0.2833154436638043, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.31258063324325003, + "recall_at_1": 0.2193761384335155, + "recall_at_10": 0.41156648451730415, + "recall_at_5": 0.3610655737704918, + "top_hit_from_holder_unanswerable": 0.0 + }, + "store": "mem0_flat [whole corpus]" }, { "by_modality": { @@ -154,7 +266,7 @@ "recall_at_1": 0.549717057396928, "recall_at_10": 0.952303961196443, "recall_at_5": 0.8884397736459175, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "conversation": { "abstention_rate_answerable": 0.0, @@ -168,7 +280,7 @@ "recall_at_1": 0.637888888888889, "recall_at_10": 0.9783333333333334, "recall_at_5": 0.9458888888888889, - "wrong_holder_confidence_unanswerable": 0.8866666666666667 + "top_hit_from_holder_unanswerable": 0.8866666666666667 }, "image": { "abstention_rate_answerable": 0.0, @@ -182,7 +294,7 @@ "recall_at_1": 0.42291666666666666, "recall_at_10": 0.6458333333333334, "recall_at_5": 0.6, - "wrong_holder_confidence_unanswerable": 0.5 + "top_hit_from_holder_unanswerable": 0.5 }, "pdf": { "abstention_rate_answerable": 0.0, @@ -196,7 +308,7 @@ "recall_at_1": 0.587467362924282, "recall_at_10": 0.9164490861618799, "recall_at_5": 0.825065274151436, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "ui_screenshot": { "abstention_rate_answerable": 0.0, @@ -210,7 +322,7 @@ "recall_at_1": 0.21875, "recall_at_10": 0.621875, "recall_at_5": 0.465625, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 }, "video": { "abstention_rate_answerable": 0.0, @@ -224,16 +336,16 @@ "recall_at_1": 0.2837209302325581, "recall_at_10": 0.7162790697674418, "recall_at_5": 0.5255813953488372, - "wrong_holder_confidence_unanswerable": 1.0 + "top_hit_from_holder_unanswerable": 1.0 } }, "closed_items": 27072, "items": 403859, "latency_seconds": { - "max": 10.518811015004758, - "mean": 0.09140969146306362, - "p50": 0.0847548475030635, - "p95": 0.10844126994816179, + "max": 18.72793256899604, + "mean": 0.09218696485981677, + "p50": 0.08754261549984221, + "p95": 0.11125843500121846, "queries": 5308 }, "overall": { @@ -248,9 +360,121 @@ "recall_at_1": 0.4980134335154827, "recall_at_10": 0.865693306010929, "recall_at_5": 0.7783071493624771, - "wrong_holder_confidence_unanswerable": 0.942436974789916 + "top_hit_from_holder_unanswerable": 0.942436974789916 }, - "store": "zep_temporal" + "store": "zep_temporal [holder scope]" + }, + { + "by_modality": { + "code": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.35004042037186744, + "hit_at_10_wilson_lower": 0.32396114426502914, + "mrr": 0.21595449820995496, + "n_answerable": 1237, + "n_unanswerable": 1237, + "ndcg_at_10": 0.24802826744611478, + "recall_at_1": 0.15683104284559418, + "recall_at_10": 0.35004042037186744, + "recall_at_5": 0.2991107518189167, + "top_hit_from_holder_unanswerable": 0.0 + }, + "conversation": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.6666666666666666, + "hit_at_10_wilson_lower": 0.5878975668678504, + "mrr": 0.42928042328042326, + "n_answerable": 150, + "n_unanswerable": 150, + "ndcg_at_10": 0.4447119968977239, + "recall_at_1": 0.2391111111111111, + "recall_at_10": 0.5912222222222222, + "recall_at_5": 0.4597777777777778, + "top_hit_from_holder_unanswerable": 0.0 + }, + "image": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.5708333333333333, + "hit_at_10_wilson_lower": 0.5075830430993526, + "mrr": 0.4262020502645502, + "n_answerable": 240, + "n_unanswerable": 240, + "ndcg_at_10": 0.45449292583858186, + "recall_at_1": 0.35208333333333336, + "recall_at_10": 0.5625, + "recall_at_5": 0.5083333333333333, + "top_hit_from_holder_unanswerable": 0.0 + }, + "pdf": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.685378590078329, + "hit_at_10_wilson_lower": 0.6516380300993592, + "mrr": 0.518995296116706, + "n_answerable": 766, + "n_unanswerable": 218, + "ndcg_at_10": 0.5587707901607282, + "recall_at_1": 0.44386422976501305, + "recall_at_10": 0.685378590078329, + "recall_at_5": 0.6174934725848564, + "top_hit_from_holder_unanswerable": 0.0 + }, + "ui_screenshot": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.11875, + "hit_at_10_wilson_lower": 0.08775049817160425, + "mrr": 0.05682539682539682, + "n_answerable": 320, + "n_unanswerable": 320, + "ndcg_at_10": 0.0711722029605294, + "recall_at_1": 0.0375, + "recall_at_10": 0.11875, + "recall_at_5": 0.08125, + "top_hit_from_holder_unanswerable": 0.0 + }, + "video": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.16279069767441862, + "hit_at_10_wilson_lower": 0.1194411092694439, + "mrr": 0.08660022148394242, + "n_answerable": 215, + "n_unanswerable": 215, + "ndcg_at_10": 0.10455353510193785, + "recall_at_1": 0.05581395348837209, + "recall_at_10": 0.16279069767441862, + "recall_at_5": 0.12093023255813953, + "top_hit_from_holder_unanswerable": 0.0 + } + }, + "closed_items": 27072, + "items": 403859, + "latency_seconds": { + "max": 0.2064202700057649, + "mean": 0.08776056289499023, + "p50": 0.08638653549860464, + "p95": 0.10662773229960291, + "queries": 5308 + }, + "overall": { + "abstention_rate_answerable": 0.0, + "abstention_rate_unanswerable": 0.0, + "hit_at_10": 0.43306010928961747, + "hit_at_10_wilson_lower": 0.41521181569494575, + "mrr": 0.29650609332986383, + "n_answerable": 2928, + "n_unanswerable": 2380, + "ndcg_at_10": 0.32645785820497475, + "recall_at_1": 0.23168260473588342, + "recall_at_10": 0.42851206739526415, + "recall_at_5": 0.3708902550091075, + "top_hit_from_holder_unanswerable": 0.0 + }, + "store": "zep_temporal [whole corpus]" } ] } \ No newline at end of file diff --git a/bench/run_external.py b/bench/run_external.py index 9387a7a..ccdcab4 100644 --- a/bench/run_external.py +++ b/bench/run_external.py @@ -12,19 +12,21 @@ from __future__ import annotations import argparse import json +import statistics from datetime import UTC, datetime from pathlib import Path from typing import Any from bench.counterpart import ( Counterpart, + SourceMeta, build_counterparts, build_items, load_item_atoms, load_source_index, ) from bench.ingest_baseline import RUNS, database_path -from bench.metrics import aggregate, score_query +from bench.metrics import aggregate, chance_recall_at_k, score_query from bench.run import Query, git_commit, holder_index, load_queries from bench.sets_common import ROOT from bench.systems import machine_facts @@ -37,6 +39,22 @@ SCOPES = (HOLDER_SCOPE, WHOLE_CORPUS_SCOPE) PROGRESS_EVERY = 500 +def chance_rows(index: dict[str, SourceMeta], queries: list[Query]) -> dict[str, Any]: + """What random guessing scores in each candidate set, so a scoped score is not read as skill.""" + per_holder: dict[str, int] = {} + for meta in index.values(): + per_holder[meta.holder] = per_holder.get(meta.holder, 0) + 1 + sizes = [per_holder[query.holder] for query in queries if query.holder in per_holder] + if not sizes: + raise SystemExit("no query holder appears in the corpus index") + return { + "recall_at_10": chance_recall_at_k(1, len(index), TOP_K), + "recall_at_10_holder_scope": sum(chance_recall_at_k(1, size, TOP_K) for size in sizes) / len(sizes), + "median_holder_sources": float(statistics.median(sizes)), + "note": "expected recall@10 of a random ranking inside the candidate set each row is allowed to see", + } + + def score_store(store: Counterpart, queries: list[Query], holder_of: dict[str, str], scope: str) -> dict[str, Any]: """Rank every query against one counterpart store and aggregate the same metrics the arms use. @@ -88,6 +106,12 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: "- extraction is lexical for both stores; Mem0's real add() and Zep's graph builder call a " "generator model, which this round does not have" ), + ( + f"- chance: random guessing scores recall@10 {report['chance']['recall_at_10']:.5f} over the whole " + f"corpus and {report['chance']['recall_at_10_holder_scope']:.4f} inside one holder's memory " + f"(median {report['chance']['median_holder_sources']:.0f} sources per holder); read each row against " + "its own floor" + ), "", ( "| store | items | closed | recall@10 | hit@10 | hit@10 Wilson 95% lower | MRR | " @@ -166,6 +190,7 @@ def main() -> int: "extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator", "machine": machine_facts(), }, + "chance": chance_rows(index, queries), "stores": [score_store(store, queries, holder_of, scope) for store in stores for scope in SCOPES], } REPORTS.mkdir(parents=True, exist_ok=True)