ZacharyZhang-NY
6de7439db6
The frozen baseline claimed the product's top-10 shared 0.0% with the fast BM25 path. That was a harness bug: gold names a corpus source (mmlongbench:PH_2016.06.08_Economy-Final.pdf) while the store names a content hash (PH_2016_06_08_Economy_Final_10e57152f44d1a2b), so the two id spaces never met and the right document, ranked first, scored as a miss. - bench/systems.py: `_corpus_ids` joins store sources through the `source_map()` the counterpart already uses, and refuses a store whose sources are not corpus files instead of silently scoring zero. - bench/systems.py: `live_product_rankings(..., as_holders=...)` asks the product as the query's own holder, which is how the counterpart is scored. - bench/run.py: the live section now reports recall@10 / MRR / hit@10 with a 95% lower bound, and probes 300 queries instead of 40 — the SQL path answers in well under a second, so the old cap was a cost estimate that expired. - bench/run.py: `--name CURRENT` writes a post-P0 report beside the frozen BASELINE without touching it, stamped with the commit and the store schema version the pass started on (the live pass migrates the store mid-run, so a plain stamp would be a lie). - bench/stamp_holders.py: the pre-P0 store holds 165,973 memories nobody owns. The corpus index states the owner, so holders, scenarios and event times are stamped in place — 165,973 atoms across 50,278 sources, 0 left unowned. Numbers, same machine, load 1.0-1.7: - CURRENT (post-P0, schema v5): no_rerank recall@10 0.565 p50 0.643s; fusion+rerank 0.453 p50 7.000s; graph 0.001. - Product as its holder: recall@10 0.870, MRR 0.740, n=173. - Counterpart: mem0_flat holder scope 0.866, whole corpus 0.412; the holder boundary is worth +0.454 with an identical ranker and identical items. Verification: pytest backend/tests bench/tests -q -> 173 passed; ruff and mypy (fresh cache) clean on every touched file.
79 lines
3.4 KiB
Python
79 lines
3.4 KiB
Python
"""Give every ingested atom the holder, scenario and event time the corpus index already states.
|
|
|
|
The pre-P0 store was built before the memory model had those columns, so it holds 165,973 memories
|
|
nobody owns. The frozen corpus index names the owner, the setting and the moment for each source —
|
|
the same facts the counterpart and the gold scopes are written from — so the store can be brought
|
|
in line without re-running ingestion.
|
|
|
|
Run: python -m bench.stamp_holders [--store bench/runs/baseline/data/ely_eye.sqlite]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
from bench.sets_common import MODALITIES, SETS, read_jsonl, record_path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
DEFAULT_STORE = REPO / "bench" / "runs" / "baseline" / "data" / "ely_eye.sqlite"
|
|
|
|
|
|
def corpus_facts() -> dict[str, tuple[str, str, str]]:
|
|
"""Absolute path -> (holder, scenario, observed_at), as the frozen corpus index states them."""
|
|
facts: dict[str, tuple[str, str, str]] = {}
|
|
for modality in MODALITIES:
|
|
for record in read_jsonl(SETS / "corpus" / f"{modality}.jsonl"):
|
|
for key in ("holder", "scenario", "observed_at"):
|
|
if not record.get(key):
|
|
raise SystemExit(f"{modality}: corpus record {record['source_id']} has no {key}")
|
|
facts[str(record_path(record).resolve())] = (
|
|
str(record["holder"]),
|
|
str(record["scenario"]),
|
|
str(record["observed_at"]),
|
|
)
|
|
if not facts:
|
|
raise SystemExit("no corpus index found, run python -m bench.build_sets first")
|
|
return facts
|
|
|
|
|
|
def stamp(store: Path) -> dict[str, int]:
|
|
"""Write the corpus facts onto every atom; the store's own source ids stay untouched."""
|
|
if not store.is_file():
|
|
raise SystemExit(f"{store}: no baseline database, run python -m bench.ingest_baseline first")
|
|
facts = corpus_facts()
|
|
with sqlite3.connect(store) as connection:
|
|
rows = connection.execute("SELECT source_id, path FROM sources").fetchall()
|
|
if not rows:
|
|
raise SystemExit(f"{store}: the sources table is empty")
|
|
unknown = sorted(str(path) for _, path in rows if str(path) not in facts)
|
|
if unknown:
|
|
raise SystemExit(f"{store}: {len(unknown)} source paths are not corpus files, first {unknown[:3]}")
|
|
stamped_atoms = 0
|
|
for source_id, path in rows:
|
|
holder, scenario, observed_at = facts[str(path)]
|
|
cursor = connection.execute(
|
|
"UPDATE atoms SET holder_id = ?, scenario = ?, event_time = ?, event_time_precision = 'instant' "
|
|
"WHERE source_id = ?",
|
|
(holder, scenario, observed_at, str(source_id)),
|
|
)
|
|
stamped_atoms += int(cursor.rowcount)
|
|
left = int(connection.execute("SELECT count(*) FROM atoms WHERE holder_id = ''").fetchone()[0])
|
|
return {"sources": len(rows), "atoms": stamped_atoms, "atoms_without_holder": left}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--store", type=Path, default=DEFAULT_STORE)
|
|
args = parser.parse_args()
|
|
counts = stamp(args.store)
|
|
print(
|
|
f"stamped {counts['atoms']} atoms across {counts['sources']} sources; "
|
|
f"{counts['atoms_without_holder']} atoms still have no holder"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|