Score the product path against corpus ids, not storage ids
The frozen baseline claimed the product's top-10 shared 0.0% with the fast BM25 path. That was a harness bug: gold names a corpus source (mmlongbench:PH_2016.06.08_Economy-Final.pdf) while the store names a content hash (PH_2016_06_08_Economy_Final_10e57152f44d1a2b), so the two id spaces never met and the right document, ranked first, scored as a miss. - bench/systems.py: `_corpus_ids` joins store sources through the `source_map()` the counterpart already uses, and refuses a store whose sources are not corpus files instead of silently scoring zero. - bench/systems.py: `live_product_rankings(..., as_holders=...)` asks the product as the query's own holder, which is how the counterpart is scored. - bench/run.py: the live section now reports recall@10 / MRR / hit@10 with a 95% lower bound, and probes 300 queries instead of 40 — the SQL path answers in well under a second, so the old cap was a cost estimate that expired. - bench/run.py: `--name CURRENT` writes a post-P0 report beside the frozen BASELINE without touching it, stamped with the commit and the store schema version the pass started on (the live pass migrates the store mid-run, so a plain stamp would be a lie). - bench/stamp_holders.py: the pre-P0 store holds 165,973 memories nobody owns. The corpus index states the owner, so holders, scenarios and event times are stamped in place — 165,973 atoms across 50,278 sources, 0 left unowned. Numbers, same machine, load 1.0-1.7: - CURRENT (post-P0, schema v5): no_rerank recall@10 0.565 p50 0.643s; fusion+rerank 0.453 p50 7.000s; graph 0.001. - Product as its holder: recall@10 0.870, MRR 0.740, n=173. - Counterpart: mem0_flat holder scope 0.866, whole corpus 0.412; the holder boundary is worth +0.454 with an identical ranker and identical items. Verification: pytest backend/tests bench/tests -q -> 173 passed; ruff and mypy (fresh cache) clean on every touched file.
This commit is contained in:
1 parent
4ab36cd5eb
commit
6de7439db6
8 files changed
+975
-68
No files matched your search
@@ -0,0 +1,78 @@
|
||||
"""Give every ingested atom the holder, scenario and event time the corpus index already states.
|
||||
|
||||
The pre-P0 store was built before the memory model had those columns, so it holds 165,973 memories
|
||||
nobody owns. The frozen corpus index names the owner, the setting and the moment for each source —
|
||||
the same facts the counterpart and the gold scopes are written from — so the store can be brought
|
||||
in line without re-running ingestion.
|
||||
|
||||
Run: python -m bench.stamp_holders [--store bench/runs/baseline/data/ely_eye.sqlite]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from bench.sets_common import MODALITIES, SETS, read_jsonl, record_path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_STORE = REPO / "bench" / "runs" / "baseline" / "data" / "ely_eye.sqlite"
|
||||
|
||||
|
||||
def corpus_facts() -> dict[str, tuple[str, str, str]]:
|
||||
"""Absolute path -> (holder, scenario, observed_at), as the frozen corpus index states them."""
|
||||
facts: dict[str, tuple[str, str, str]] = {}
|
||||
for modality in MODALITIES:
|
||||
for record in read_jsonl(SETS / "corpus" / f"{modality}.jsonl"):
|
||||
for key in ("holder", "scenario", "observed_at"):
|
||||
if not record.get(key):
|
||||
raise SystemExit(f"{modality}: corpus record {record['source_id']} has no {key}")
|
||||
facts[str(record_path(record).resolve())] = (
|
||||
str(record["holder"]),
|
||||
str(record["scenario"]),
|
||||
str(record["observed_at"]),
|
||||
)
|
||||
if not facts:
|
||||
raise SystemExit("no corpus index found, run python -m bench.build_sets first")
|
||||
return facts
|
||||
|
||||
|
||||
def stamp(store: Path) -> dict[str, int]:
|
||||
"""Write the corpus facts onto every atom; the store's own source ids stay untouched."""
|
||||
if not store.is_file():
|
||||
raise SystemExit(f"{store}: no baseline database, run python -m bench.ingest_baseline first")
|
||||
facts = corpus_facts()
|
||||
with sqlite3.connect(store) as connection:
|
||||
rows = connection.execute("SELECT source_id, path FROM sources").fetchall()
|
||||
if not rows:
|
||||
raise SystemExit(f"{store}: the sources table is empty")
|
||||
unknown = sorted(str(path) for _, path in rows if str(path) not in facts)
|
||||
if unknown:
|
||||
raise SystemExit(f"{store}: {len(unknown)} source paths are not corpus files, first {unknown[:3]}")
|
||||
stamped_atoms = 0
|
||||
for source_id, path in rows:
|
||||
holder, scenario, observed_at = facts[str(path)]
|
||||
cursor = connection.execute(
|
||||
"UPDATE atoms SET holder_id = ?, scenario = ?, event_time = ?, event_time_precision = 'instant' "
|
||||
"WHERE source_id = ?",
|
||||
(holder, scenario, observed_at, str(source_id)),
|
||||
)
|
||||
stamped_atoms += int(cursor.rowcount)
|
||||
left = int(connection.execute("SELECT count(*) FROM atoms WHERE holder_id = ''").fetchone()[0])
|
||||
return {"sources": len(rows), "atoms": stamped_atoms, "atoms_without_holder": left}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--store", type=Path, default=DEFAULT_STORE)
|
||||
args = parser.parse_args()
|
||||
counts = stamp(args.store)
|
||||
print(
|
||||
f"stamped {counts['atoms']} atoms across {counts['sources']} sources; "
|
||||
f"{counts['atoms_without_holder']} atoms still have no holder"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user