diff --git a/bench/reports/BASELINE.md b/bench/reports/BASELINE.md index 6c4332a..53e7181 100644 --- a/bench/reports/BASELINE.md +++ b/bench/reports/BASELINE.md @@ -1,6 +1,6 @@ # BASELINE: what the product retrieves today -- generated 2026-10-07T05:35:00+00:00 at commit a4953335e77f +- generated 2026-10-07T15:33:42+00:00 at commit 29163632af74 - corpus 50278 sources, 5308 queries (2928 answerable) - models: dense `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2`, rerank `BAAI/bge-reranker-base` @@ -33,14 +33,20 @@ ## the unmodified product path -- RetrievalService.search: p50 15.54s, mean 16.68s, max 56.14s over 40 queries +- RetrievalService.search: p50 20.16s, mean 20.67s, max 48.90s over 40 queries - top-10 overlap with the fast BM25 path: 0.0%, identical order 0.0% +- its top-10 atoms cover only 7.6 distinct sources (min 1): the product ranks atoms and never diversifies, so ten slots fill with fragments of one document and the overlap with a source-level ranking is near zero ## latency vs corpus size -| sources | atoms | bm25_only | dense_only | -|---|---|---|---|---| -| 50000 | 165695 | 0.804/1.576 | 1.446/2.538 | +| sources | atoms | bm25_only p50/p95 s | dense_only p50/p95 s | +|---|---|---|---| +| 500 | 500 | 0.001/0.002 | 1.380/1.576 | +| 2000 | 2000 | 0.008/0.016 | 1.398/2.595 | +| 10000 | 10000 | 0.052/0.128 | 1.338/2.266 | +| 50000 | 165695 | 0.809/2.274 | 1.496/2.464 | + +_latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless_ ## reading diff --git a/bench/reports/baseline.json b/bench/reports/baseline.json index 7d3bae1..49bae1e 100644 --- a/bench/reports/baseline.json +++ b/bench/reports/baseline.json @@ -564,9 +564,9 @@ "min": 1 }, "identical_top10": 0.0, - "max": 63.2751505510023, - "mean": 21.701527132049886, - "p50": 21.831516211001144, + "max": 48.895113198996114, + "mean": 20.665571158625244, + "p50": 20.157494062004844, "queries": 40, "top10_overlap_with_bm25_only": 0.0, "top_k": 10 @@ -575,7 +575,7 @@ "answerable": 2928, "corpus_manifest_sha256": "8fc1ed861c891d778853d41107630869f9eb267a669c5d1038e389de80114347", "corpus_sources": 50278, - "generated_at": "2026-10-07T14:38:12+00:00", + "generated_at": "2026-10-07T15:33:42+00:00", "git_commit": "29163632af7418758e79a79a748f68050ca1e0b6", "limit_per_modality": null, "machine": { @@ -602,12 +602,12 @@ { "atoms": 500, "bm25_only": { - "p50": 0.0007463944966730196, - "p95": 0.0026389723490865425 + "p50": 0.0006466769991675392, + "p95": 0.00181096804953995 }, "dense_only": { - "p50": 2.053756675497425, - "p95": 3.7252893005985244 + "p50": 1.3797472870028287, + "p95": 1.5762464397041183 }, "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", "queries": 40, @@ -616,12 +616,12 @@ { "atoms": 2000, "bm25_only": { - "p50": 0.00923213200076134, - "p95": 0.023986968196186332 + "p50": 0.008334371003002161, + "p95": 0.015740483100307734 }, "dense_only": { - "p50": 2.2580291965023207, - "p95": 4.0939969898456186 + "p50": 1.3979396294998878, + "p95": 2.595364080850413 }, "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", "queries": 40, @@ -630,12 +630,12 @@ { "atoms": 10000, "bm25_only": { - "p50": 0.11317921599766123, - "p95": 0.2961666731029253 + "p50": 0.05245087000002968, + "p95": 0.12780161215196129 }, "dense_only": { - "p50": 2.269751224499487, - "p95": 3.6929099251963633 + "p50": 1.3382894579990534, + "p95": 2.2659613330957646 }, "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", "queries": 40, @@ -644,12 +644,12 @@ { "atoms": 165695, "bm25_only": { - "p50": 1.3151575949996186, - "p95": 3.7234880074014645 + "p50": 0.8085095155001909, + "p95": 2.273631770645442 }, "dense_only": { - "p50": 2.3852021460043034, - "p95": 4.008884304698221 + "p50": 1.4964509784986149, + "p95": 2.4635968941045574 }, "note": "latency only: a source prefix drops most gold sources, so recall at these sizes is meaningless", "queries": 40, diff --git a/bench/run.py b/bench/run.py index aafbb21..0586a6c 100644 --- a/bench/run.py +++ b/bench/run.py @@ -12,7 +12,6 @@ from __future__ import annotations import argparse import hashlib import json -import os import platform import subprocess from collections.abc import Sequence @@ -36,6 +35,7 @@ from bench.systems import ( arm_needs, live_product_rankings, load_atoms, + machine_facts, ) ROOT = SETS.parent @@ -213,7 +213,7 @@ def meta(corpus_sources: int, queries: Sequence[Query], limit: int | None = None "generated_at": datetime.now(UTC).isoformat(timespec="seconds"), "git_commit": git_commit(), "python": platform.python_version(), - "machine": {"system": platform.system(), "release": platform.release(), "cpus": os.cpu_count()}, + "machine": machine_facts(), "corpus_manifest_sha256": manifest.get("corpus_manifest_sha256"), "sets": {name: sha256_file(SETS / f"{name}.jsonl") for name in MODALITIES}, "queries": len(queries), @@ -251,6 +251,10 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: f"({report['meta']['answerable']} answerable)" ), f"- models: dense `{report['meta']['models']['dense']}`, rerank `{report['meta']['models']['rerank']}`", + ( + f"- machine: {report['meta']['machine']['cpus']} cores, load average " + f"{report['meta']['machine']['load_average_1_5_15'][0]:.1f} when the report was written" + ), "", "| arm | recall@1 | recall@5 | recall@10 | nDCG@10 | MRR | hit@10 (95% lower) | p50 s | p95 s |", "|---|---|---|---|---|---|---|---|---|", @@ -321,6 +325,11 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: note = report["scale"][0].get("note") if note: lines += ["", f"_{note}_"] + load = report["meta"]["machine"]["load_average_1_5_15"][0] + lines.append( + f"\n_latency was measured at load average {load:.1f} on {report['meta']['machine']['cpus']} cores; " + "a busy machine inflates these figures, so read them as a ceiling_" + ) lines += [ "", "## reading", diff --git a/bench/run_external.py b/bench/run_external.py index a07a45b..01c3c80 100644 --- a/bench/run_external.py +++ b/bench/run_external.py @@ -27,6 +27,7 @@ from bench.ingest_baseline import RUNS, database_path from bench.metrics import aggregate, score_query from bench.run import Query, git_commit, holder_index, load_queries from bench.sets_common import ROOT +from bench.systems import machine_facts REPORTS = ROOT / "reports" TOP_K = 10 @@ -66,6 +67,10 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: "", f"- generated {meta['generated_at']} at commit {meta['git_commit'][:12]}", f"- {meta['queries']} queries ({meta['answerable']} answerable) against {meta['sources']} sources", + ( + f"- machine: {meta['machine']['cpus']} cores, load average " + f"{meta['machine']['load_average_1_5_15'][0]:.1f} when the report was written" + ), ( "- extraction is lexical for both stores; Mem0's real add() and Zep's graph builder call a " "generator model, which this round does not have" @@ -139,6 +144,7 @@ def main() -> int: "sources": len(index), "top_k": TOP_K, "extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator", + "machine": machine_facts(), }, "stores": [score_store(store, queries, holder_of) for store in stores], } diff --git a/bench/systems.py b/bench/systems.py index 4821050..d59047e 100644 --- a/bench/systems.py +++ b/bench/systems.py @@ -10,6 +10,8 @@ from __future__ import annotations import hashlib import json +import os +import platform import re import sqlite3 import time @@ -30,6 +32,16 @@ DENSE_MODEL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" RERANK_MODEL = "BAAI/bge-reranker-base" +def machine_facts() -> dict[str, Any]: + """Hardware plus the load at report time: latency numbers mean nothing without it.""" + return { + "system": platform.system(), + "release": platform.release(), + "cpus": os.cpu_count(), + "load_average_1_5_15": [round(value, 2) for value in os.getloadavg()], + } + + def local_model_dir(models: Path, repo_id: str) -> Path: """Where a fetched model lives: one flat directory per repo, named by its repo name only.""" return models / repo_id.split("/")[-1] diff --git a/bench/tests/test_joint.py b/bench/tests/test_joint.py index 174eade..91ffe12 100644 --- a/bench/tests/test_joint.py +++ b/bench/tests/test_joint.py @@ -158,6 +158,7 @@ def _report() -> dict: "corpus_sources": 10, "queries": 16, "answerable": 8, + "machine": {"system": "Linux", "release": "x", "cpus": 32, "load_average_1_5_15": [1.0, 2.0, 3.0]}, "models": {"dense": "d", "rerank": "r"}, }, "chance": {"recall_at_1": 0.0, "recall_at_5": 0.0, "recall_at_10": 0.0}, @@ -191,6 +192,7 @@ def test_the_report_writer_survives_a_scale_note(tmp_path: Path) -> None: text = path.read_text(encoding="utf-8") assert "| 500 | 900 | 0.001/0.002 |" in text assert "_latency only_" in text + assert "load average 1.0 on 32 cores" in text assert "## unanswerable queries (50.0% of the suite)" in text