Score the product path against corpus ids, not storage ids
The frozen baseline claimed the product's top-10 shared 0.0% with the fast BM25 path. That was a harness bug: gold names a corpus source (mmlongbench:PH_2016.06.08_Economy-Final.pdf) while the store names a content hash (PH_2016_06_08_Economy_Final_10e57152f44d1a2b), so the two id spaces never met and the right document, ranked first, scored as a miss. - bench/systems.py: `_corpus_ids` joins store sources through the `source_map()` the counterpart already uses, and refuses a store whose sources are not corpus files instead of silently scoring zero. - bench/systems.py: `live_product_rankings(..., as_holders=...)` asks the product as the query's own holder, which is how the counterpart is scored. - bench/run.py: the live section now reports recall@10 / MRR / hit@10 with a 95% lower bound, and probes 300 queries instead of 40 — the SQL path answers in well under a second, so the old cap was a cost estimate that expired. - bench/run.py: `--name CURRENT` writes a post-P0 report beside the frozen BASELINE without touching it, stamped with the commit and the store schema version the pass started on (the live pass migrates the store mid-run, so a plain stamp would be a lie). - bench/stamp_holders.py: the pre-P0 store holds 165,973 memories nobody owns. The corpus index states the owner, so holders, scenarios and event times are stamped in place — 165,973 atoms across 50,278 sources, 0 left unowned. Numbers, same machine, load 1.0-1.7: - CURRENT (post-P0, schema v5): no_rerank recall@10 0.565 p50 0.643s; fusion+rerank 0.453 p50 7.000s; graph 0.001. - Product as its holder: recall@10 0.870, MRR 0.740, n=173. - Counterpart: mem0_flat holder scope 0.866, whole corpus 0.412; the holder boundary is worth +0.454 with an identical ranker and identical items. Verification: pytest backend/tests bench/tests -q -> 173 passed; ruff and mypy (fresh cache) clean on every touched file.
This commit is contained in:
1 parent
4ab36cd5eb
commit
6de7439db6
8 files changed
+975
-68
No files matched your search
+46
-10
@@ -18,11 +18,14 @@ import time
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from bench.ingest_baseline import source_map
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ely_eye.db import Database
|
||||
from bench.sets_common import ROOT
|
||||
|
||||
RUNS = ROOT / "runs" / "baseline"
|
||||
@@ -423,34 +426,67 @@ def _best_atom_text(atoms: Sequence[Atom], source_ids: Sequence[str]) -> list[tu
|
||||
return [(source_id, longest.get(source_id, "")) for source_id in source_ids]
|
||||
|
||||
|
||||
def _corpus_ids(database: Database) -> dict[str, str]:
|
||||
"""Join the store's content-hash source ids to the ids the frozen sets call a source by.
|
||||
|
||||
Without this join the product's hits and the gold answers share no name, and a correct answer
|
||||
scores zero — which is what an earlier report recorded as "0% overlap with the fast path".
|
||||
"""
|
||||
mapping = source_map()
|
||||
ids: dict[str, str] = {}
|
||||
unknown: list[str] = []
|
||||
for source in database.list_sources():
|
||||
corpus_id = mapping.get(str(Path(source.path).resolve()))
|
||||
if corpus_id is None:
|
||||
unknown.append(source.source_id)
|
||||
else:
|
||||
ids[source.source_id] = corpus_id
|
||||
if unknown:
|
||||
raise SystemExit(f"{database.path}: {len(unknown)} store sources are not corpus files, first {unknown[:3]}")
|
||||
return ids
|
||||
|
||||
|
||||
def live_product_rankings(
|
||||
db_home: Path, queries: Sequence[str], top_k: int = 10
|
||||
db_home: Path,
|
||||
queries: Sequence[str],
|
||||
top_k: int = 10,
|
||||
as_holders: Sequence[str | None] | None = None,
|
||||
) -> tuple[list[list[str]], list[int], list[float]]:
|
||||
"""Rankings from the unmodified RetrievalService, with per-query wall time.
|
||||
|
||||
The second list counts how many distinct sources the product's top-k atoms came
|
||||
from: the product ranks atoms and never diversifies, so ten slots can be filled by
|
||||
fragments of one document, and that is invisible once the hits are collapsed.
|
||||
Rankings carry the benchmark's source ids, so they are comparable with the arms and with gold.
|
||||
`as_holders` asks each query as its own holder, which is how a transactive memory is used: the
|
||||
asker does not know where the answer is, only who might.
|
||||
|
||||
The second list counts how many distinct sources the product's top-k atoms came from: the
|
||||
product ranks atoms and never diversifies, so ten slots can be filled by fragments of one
|
||||
document, and that is invisible once the hits are collapsed.
|
||||
"""
|
||||
from ely_eye.config import Settings
|
||||
from ely_eye.db import Database
|
||||
from ely_eye.retrieval import RetrievalService
|
||||
from ely_eye.schemas import SearchFilters
|
||||
|
||||
if as_holders is not None and len(as_holders) != len(queries):
|
||||
raise SystemExit(f"{len(as_holders)} holders for {len(queries)} queries")
|
||||
settings = Settings(home=db_home)
|
||||
service = RetrievalService(settings=settings, database=Database(settings))
|
||||
database = Database(settings)
|
||||
service = RetrievalService(settings=settings, database=database)
|
||||
corpus_ids = _corpus_ids(database)
|
||||
rankings: list[list[str]] = []
|
||||
distinct: list[int] = []
|
||||
timings: list[float] = []
|
||||
for query in queries:
|
||||
for index, query in enumerate(queries):
|
||||
holder = as_holders[index] if as_holders is not None else None
|
||||
started = time.perf_counter()
|
||||
hits = service.search(query, top_k=top_k).hits
|
||||
hits = service.search(query, top_k=top_k, filters=SearchFilters(as_holder=holder)).hits
|
||||
timings.append(time.perf_counter() - started)
|
||||
# The product ranks atoms; collapse to sources by first (best) hit so the arms stay comparable.
|
||||
sources: list[str] = []
|
||||
for hit in hits:
|
||||
source_id = hit.atom.source_id
|
||||
source_id = corpus_ids[hit.atom.source_id]
|
||||
if source_id not in sources:
|
||||
sources.append(source_id)
|
||||
distinct.append(len({hit.atom.source_id for hit in hits}))
|
||||
distinct.append(len({corpus_ids[hit.atom.source_id] for hit in hits}))
|
||||
rankings.append(sources[:top_k])
|
||||
return rankings, distinct, timings
|
||||
Reference in new issue
Block a user