Record machine load with every latency number
Latency measured while another job hogs the cores is not comparable: the scale curve moved from 1.315s to 0.809s at 50,000 sources between two runs of the same code. - bench/systems.py: machine_facts() reports cpus plus the 1/5/15 minute load average. - bench/run.py, bench/run_external.py: both reports carry it, and BASELINE.md labels the latency table a ceiling when the machine was busy. Verification: pytest bench/tests -q -> 60 passed; ruff check bench/ -> all checks passed; mypy (fresh cache) -> no issues in 25 source files.
This commit is contained in:
1 parent
61f6506104
commit
c4e2982506
6 files changed
+62
-27
No files matched your search
@@ -27,6 +27,7 @@ from bench.ingest_baseline import RUNS, database_path
|
||||
from bench.metrics import aggregate, score_query
|
||||
from bench.run import Query, git_commit, holder_index, load_queries
|
||||
from bench.sets_common import ROOT
|
||||
from bench.systems import machine_facts
|
||||
|
||||
REPORTS = ROOT / "reports"
|
||||
TOP_K = 10
|
||||
@@ -66,6 +67,10 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
|
||||
"",
|
||||
f"- generated {meta['generated_at']} at commit {meta['git_commit'][:12]}",
|
||||
f"- {meta['queries']} queries ({meta['answerable']} answerable) against {meta['sources']} sources",
|
||||
(
|
||||
f"- machine: {meta['machine']['cpus']} cores, load average "
|
||||
f"{meta['machine']['load_average_1_5_15'][0]:.1f} when the report was written"
|
||||
),
|
||||
(
|
||||
"- extraction is lexical for both stores; Mem0's real add() and Zep's graph builder call a "
|
||||
"generator model, which this round does not have"
|
||||
@@ -139,6 +144,7 @@ def main() -> int:
|
||||
"sources": len(index),
|
||||
"top_k": TOP_K,
|
||||
"extraction": "lexical (see bench/counterpart.py); LLM extraction deferred with the generator",
|
||||
"machine": machine_facts(),
|
||||
},
|
||||
"stores": [score_store(store, queries, holder_of) for store in stores],
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user