ZacharyZhang-NY/ely-eye
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep the vectors beside the atoms instead of rebuilding them per query

Browse Source 
The dense leg used to be a pair of files next to the database: a full .npy
re-encoded from every atom, re-normalized on every query, and stale the moment
anything was ingested. It is now a vec0 table inside the same SQLite file,
written in the same transaction as the atom it describes, so a memory is
queryable the instant it is remembered.

- vector_index.py maintains the table; the writer inserts, the reader never
  rebuilds. A repair scan (ely-eye build-dense-index) catches what a crash hid,
  and it is not on the search path.
- Ranking stays in the engine; every boundary (closed, retired, not yet true,
  another holder's private memory, the mounted cartridge) is checked once against
  the atoms table afterwards. A visibility subquery inside the KNN cost 48.9s on
  153,857 vectors; the same check over 200 candidates costs 0.2ms. When the
  boundaries reject enough candidates the ask widens once, never twice.
- The Go mirror and the web panel carry the new status fields.
- bench/dense_scale.py and bench/quant_probe.py measure the leg at corpus size;
  bench/scale_check.py now works on a copy so the frozen baseline stays frozen.

Measured on the frozen 165,973-atom corpus. bench/reports/DENSE.md, QUANT.md and
SCALE.md carry the numbers and the machine they were taken on.

Gates: 285 pytest, ruff clean, pyright 0 on the touched files, go build/vet/test
and gofmt clean, frontend build and eslint clean. mypy is 25 findings against the
parent's 24: the new one is "Skipping analyzing sqlite_vec: missing library stubs",
the same class as the two existing qwen_vl_utils lines, and the repo keeps no
type: ignore anywhere.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-08 14:05:27 -04:00
1 parent be08e7bf9c
commit cd9851a12c
26 files changed
+1602 -118
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bench/dense_scale.py
+318
View File
@@ -0,0 +1,318 @@
"""What the dense leg costs at the size the library actually reaches, and what a million would.
The baseline store holds 165,973 atoms. This check builds the product's own vector table inside
a copy of that store, times real searches, proves the engine's answers match a brute-force scan,
and then states what one million memories would cost on the same machine — with the assumption
written down instead of implied.
"""
from __future__ import annotations
import argparse
import json
import shutil
import sqlite3
import time
from collections.abc import Sequence
from pathlib import Path
from typing import TYPE_CHECKING, Any
import numpy as np
from bench.query_set import load_queries, scale_probes
from bench.report import git_commit
from bench.run import latency
from bench.systems import DENSE_MODEL, MODELS, machine_facts
if TYPE_CHECKING:
from ely_eye.db import Database
DEFAULT_HOME = Path("bench/runs/baseline")
DEFAULT_SCRATCH = Path("bench/runs/dense_scale")
MIN_ATOMS = 100_000
TOP_K = 10
CORRECTNESS_PROBES = 40
EXTRAPOLATE_TO = 1_000_000
WRITE_PROBES = 50
def _one_write(index: Any, atom_id: str) -> float:
"""Time one memory going into an index that already holds the whole corpus."""
began = time.perf_counter()
index.sync_atoms([atom_id])
return time.perf_counter() - began
def vector_footprint(database: Any, table: str) -> tuple[int, str]:
"""What the vector table costs on disk, and how that was counted.
The database file itself is a poor measure: free pages absorb a table this size, so the file
can stay exactly the same length while the table grows. dbstat counts the pages the vec0
shadow tables actually hold, and it works without the vec0 extension loaded.
"""
with database.connect() as connection:
try:
pages = connection.execute(
"SELECT COALESCE(SUM(pgsize), 0) FROM dbstat WHERE name LIKE ?", (f"{table}%",)
).fetchone()[0]
if int(pages):
return int(pages), "dbstat page sizes over the vec0 shadow tables"
except sqlite3.OperationalError:
pass
payload = connection.execute(
"SELECT COALESCE(SUM(length(embedding)), 0) FROM vec_atoms"
).fetchone()[0]
return int(payload), "vector bytes only, this sqlite3 has no dbstat"
def copy_store(home: Path, scratch: Path) -> tuple[Path, float]:
"""Work on a copy: the baseline store is frozen evidence and must not be migrated in place."""
source = home / "data" / "ely_eye.sqlite"
if not source.is_file():
raise SystemExit(f"{source}: no product database, build it with python -m bench.ingest_baseline")
data = scratch / "data"
data.mkdir(parents=True, exist_ok=True)
began = time.perf_counter()
for suffix in ("", "-wal", "-shm"):
if (path := Path(str(source) + suffix)).is_file():
shutil.copyfile(path, data / (source.name + suffix))
return scratch, time.perf_counter() - began
def stored_vectors(database: Database) -> tuple[list[str], np.ndarray]:
"""Every vector the engine holds, for a scan it can be checked against."""
with database.connect() as connection:
rows = connection.execute(
"""
SELECT atom_id, embedding FROM vec_atoms ORDER BY atom_id
"""
).fetchall()
if not rows:
raise SystemExit("the vector table is empty, so there is nothing to measure")
ids = [str(row["atom_id"]) for row in rows]
matrix = np.stack([np.frombuffer(row["embedding"], dtype=np.float32) for row in rows])
return ids, matrix
def agreement(ids: list[str], matrix: np.ndarray, probes: Sequence[tuple[np.ndarray, list[str]]]) -> dict[str, Any]:
"""How often the engine's top ten agrees with a full scan, counting a tie as a tie."""
if not probes:
raise SystemExit("correctness needs at least one probe")
matched = 0
compared = 0
for vector, returned in probes:
cosine = matrix @ vector
order = np.argsort(-cosine, kind="stable")
threshold = cosine[order[TOP_K - 1]]
correct = {ids[index] for index in order if cosine[index] >= threshold}
matched += sum(1 for atom_id in returned if atom_id in correct)
compared += min(len(returned), TOP_K)
return {
"probes": len(probes),
"top_k": TOP_K,
"tie_aware_agreement": matched / compared if compared else 0.0,
"note": "a returned atom counts when its true cosine is at least the k-th best, so ties cannot fail the check",
}
def measure(home: Path, scratch: Path, probes: int, model_dir: Path) -> dict[str, Any]:
from ely_eye.config import Settings
from ely_eye.db import Database
from ely_eye.embeddings import EmbeddingService
from ely_eye.vector_index import VectorIndex, expected_vectors
copy_home, copy_seconds = copy_store(home, scratch)
settings = Settings(
home=copy_home,
dense_embeddings_enabled=True,
embedding_model_dir=model_dir,
embedding_model_id=DENSE_MODEL,
)
database = Database(settings)
atoms = database.atom_count()
if atoms < MIN_ATOMS:
raise SystemExit(f"{database.path}: {atoms} atoms, this check needs at least {MIN_ATOMS}")
embeddings = EmbeddingService(settings, database)
index = VectorIndex(settings, database, embeddings)
print(f"{atoms} atoms copied in {copy_seconds:.1f}s, encoding with {DENSE_MODEL}", flush=True)
began = time.perf_counter()
index.sync()
build_seconds = time.perf_counter() - began
vec_bytes, bytes_method = vector_footprint(database, "vec_atoms")
print(f"index built in {build_seconds:.1f}s", flush=True)
began = time.perf_counter()
index.sync()
repair_seconds = time.perf_counter() - began
print(f"nothing to repair, the scan took {repair_seconds:.1f}s", flush=True)
queries = scale_probes(load_queries(), probes)
index.search(embeddings.embed_query(queries[0].text), TOP_K)
search_times: list[float] = []
encode_times: list[float] = []
returned: list[tuple[str, list[str]]] = []
for query in queries:
began = time.perf_counter()
vector = embeddings.embed_query(query.text)
encode_times.append(time.perf_counter() - began)
began = time.perf_counter()
hits = index.search(vector, TOP_K)
search_times.append(time.perf_counter() - began)
returned.append((query.text, list(hits)))
if len(search_times) % 100 == 0:
print(f"searched {len(search_times)}/{len(queries)}, p50 {sorted(search_times)[len(search_times) // 2] * 1000:.1f}ms", flush=True)
ids, matrix = stored_vectors(database)
print(f"{len(ids)} vectors read back, checking the engine against a brute-force scan", flush=True)
checked_probes = []
for query in queries[:CORRECTNESS_PROBES]:
vector = embeddings.embed_query(query.text)
checked_probes.append((vector, list(index.search(vector, TOP_K))))
checked = agreement(ids, matrix, checked_probes)
print(f"agreement {checked['tie_aware_agreement']:.3f} over {checked['probes']} probes", flush=True)
written = [hit[0] for _, hit in returned[:WRITE_PROBES] if hit]
writes = latency([_one_write(index, atom_id) for atom_id in written])
print(f"{len(written)} writes into a full index, p50 {writes['p50'] * 1000:.0f}ms", flush=True)
facts = index.stats()
encoded = int(facts["vectors"])
per_vector = vec_bytes / encoded if encoded else 0.0
texts_per_second = encoded / build_seconds if build_seconds else 0.0
timing = latency(search_times)
return {
"commit": git_commit(),
"database": str(database.path),
"atoms": atoms,
"memories_with_words": expected_vectors(database),
"vectors": encoded,
"pending": facts["pending"],
"dim": facts["dim"],
"model": facts["model"],
"copy_seconds": copy_seconds,
"build_seconds": build_seconds,
"texts_per_second": texts_per_second,
"repair_seconds": repair_seconds,
"writes": len(written),
"write_latency": writes,
"probes": len(queries),
"probe_modalities": sorted({query.modality for query in queries}),
"latency": timing,
"encode_latency": latency(encode_times),
"correctness": checked,
"bytes_per_vector": per_vector,
"vector_bytes": vec_bytes,
"bytes_method": bytes_method,
"extrapolation": {
"to": EXTRAPOLATE_TO,
"assumption": (
"sqlite-vec 0.1.9 scans every vector, so latency and bytes grow linearly with the "
"count; the model, the machine and the thread budget stay the same"
),
"p50_seconds": timing["p50"] * EXTRAPOLATE_TO / atoms,
"p95_seconds": timing["p95"] * EXTRAPOLATE_TO / atoms,
"write_p50_seconds": writes["p50"] * EXTRAPOLATE_TO / atoms,
"gibibytes": per_vector * EXTRAPOLATE_TO / 1024**3,
"build_seconds": EXTRAPOLATE_TO / texts_per_second if texts_per_second else None,
},
"machine": machine_facts(),
}
def write_markdown(report: dict[str, Any], path: Path) -> None:
timing = report["latency"]
encode = report["encode_latency"]
write = report["write_latency"]
machine = report["machine"]
future = report["extrapolation"]
path.write_text(
"\n".join(
[
f"# DENSE: the vector leg at corpus size (commit {report['commit'][:9]})",
"",
f"- database: `{report['database']}`",
(
f"- atoms: {report['atoms']}, memories with words: {report['memories_with_words']}, "
f"vectors: {report['vectors']}, pending: {report['pending']}"
),
f"- model: `{report['model']}`, width {report['dim']}",
(
f"- build: {report['build_seconds']:.1f}s for {report['vectors']} memories "
f"({report['texts_per_second']:.0f} per second)"
),
(
f"- writing one memory into a full index: p50 {write['p50'] * 1000:.0f}ms · "
f"p95 {write['p95'] * 1000:.0f}ms over {write['queries']} writes"
),
f"- the repair scan (`ely-eye build-dense-index`) over a full index: {report['repair_seconds']:.1f}s, and it is not on the search path",
(
f"- search p50 {timing['p50'] * 1000:.1f}ms · p95 {timing['p95'] * 1000:.1f}ms · "
f"max {timing['max'] * 1000:.1f}ms over {timing['queries']} queries"
),
(
f"- of which encoding the question: p50 {encode['p50'] * 1000:.1f}ms · "
f"p95 {encode['p95'] * 1000:.1f}ms"
),
(
f"- against a brute-force scan: {report['correctness']['tie_aware_agreement']:.3f} "
f"of the top {report['correctness']['top_k']} over {report['correctness']['probes']} probes "
f"({report['correctness']['note']})"
),
(
f"- bytes per stored vector: {report['bytes_per_vector']:.0f} "
f"({report['vector_bytes']} bytes for {report['vectors']} vectors, {report['bytes_method']})"
),
(
f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, "
f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"
),
"",
"## One million memories on this machine",
"",
f"- p50 {future['p50_seconds'] * 1000:.0f}ms · p95 {future['p95_seconds'] * 1000:.0f}ms",
f"- vector table: {future['gibibytes']:.1f} GiB",
f"- first build: {future['build_seconds'] / 60:.1f} minutes",
f"- writing one memory: {future['write_p50_seconds'] * 1000:.0f}ms — a vec0 delete by id scans the table, so this grows with the count too",
f"- assumption: {future['assumption']}",
"",
(
"_Latency is wall time per search on the machine named above with the thread budget the "
"benchmark pins; a run on a loaded machine is not comparable._"
),
"",
]
),
encoding="utf-8",
)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--home", type=Path, default=DEFAULT_HOME)
parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH)
parser.add_argument("--probes", type=int, default=300)
parser.add_argument("--model-dir", type=Path, default=MODELS / "paraphrase-multilingual-MiniLM-L12-v2")
parser.add_argument("--out", type=Path, default=Path("bench/reports/DENSE.md"))
parser.add_argument("--json-out", type=Path, default=Path("bench/reports/dense.json"))
args = parser.parse_args()
if args.probes <= 0:
raise SystemExit(f"--probes must be positive, got {args.probes}")
if not args.model_dir.is_dir():
raise SystemExit(f"{args.model_dir}: no embedding snapshot, run python -m bench.fetch_models")
report = measure(args.home, args.scratch, args.probes, args.model_dir)
args.json_out.parent.mkdir(parents=True, exist_ok=True)
args.json_out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
write_markdown(report, args.out)
timing = report["latency"]
print(
f"{report['vectors']} vectors: build {report['build_seconds']:.1f}s, "
f"search p50 {timing['p50'] * 1000:.1f}ms p95 {timing['p95'] * 1000:.1f}ms, "
f"agreement {report['correctness']['tie_aware_agreement']:.3f}",
flush=True,
)
if __name__ == "__main__":
main()
bench/quant_probe.py
+332
View File
@@ -0,0 +1,332 @@
"""What each way of storing a vector actually buys: latency, recall past the ties, and bytes.
The product stores float32 today. This check answers, on the real corpus and with the thread
budget the benchmark pins, whether int8, a sign bit, or a shorter vector would buy anything —
and where the crossover is, so the answer does not have to be guessed at a million memories.
"""
from __future__ import annotations
import argparse
import json
import random
import sqlite3
import time
from pathlib import Path
from typing import Any
import numpy as np
import sqlite_vec
from bench.report import git_commit
from bench.run import latency
from bench.systems import machine_facts
DEFAULT_VECTORS = Path(
"bench/runs/baseline/index/paraphrase-multilingual-MiniLM-L12-v2-7a66920843343254.npy"
)
PROBES = 200
TOP = 10
K = 10
COARSE = 200
SEED = 7
TRUNCATIONS = (192, 128, 64)
BYTES_PER_VECTOR = {"float32": 1536, "int8": 384, "bit": 48}
def load_matrix(path: Path) -> np.ndarray:
if not path.is_file():
raise SystemExit(f"{path}: no frozen vector matrix, build it with python -m bench.run")
matrix = np.load(path)
if matrix.ndim != 2:
raise SystemExit(f"{path}: expected a matrix of vectors, got shape {matrix.shape}")
return matrix
def tie_aware_hits(cosine: np.ndarray, threshold: np.ndarray, returned: list[int], probe: int) -> int:
"""The corpus holds byte-identical atoms, so the top-K id set is ambiguous: a returned atom
counts when its true cosine is at least the k-th best, which is what a caller would accept."""
return sum(1 for row in returned[:K] if cosine[row][probe] >= threshold[probe] - 1e-6)
def run_arm(
name: str,
arm: Any,
count: int,
warm: int,
cosine: np.ndarray,
threshold: np.ndarray,
) -> dict[str, Any]:
for index in range(warm):
arm(index)
times: list[float] = []
hits = 0
for index in range(count):
began = time.perf_counter()
returned = arm(index)
times.append(time.perf_counter() - began)
hits += tie_aware_hits(cosine, threshold, returned, index)
timing = latency(times)
return {
"arm": name,
"queries": count,
"p50_ms": round(timing["p50"] * 1000, 1),
"p95_ms": round(timing["p95"] * 1000, 1),
f"recall_at_{K}": round(hits / (count * K), 3),
}
def vec_table(kind: str, count: int, dim: int, payload: Any) -> sqlite3.Connection:
connection = sqlite3.connect(":memory:")
connection.enable_load_extension(True)
sqlite_vec.load(connection)
if kind == "float":
connection.executescript(
f"""
CREATE VIRTUAL TABLE vv USING vec0(
embedding float[{dim}] distance_metric=cosine, atom_id text, holder_id text
);
"""
)
insert = """
INSERT INTO vv(rowid, embedding, atom_id, holder_id) VALUES (?, ?, ?, ?)
"""
elif kind == "int8":
connection.executescript(
f"""
CREATE VIRTUAL TABLE vv USING vec0(
embedding int8[{dim}], atom_id text, holder_id text
);
"""
)
insert = """
INSERT INTO vv(rowid, embedding, atom_id, holder_id) VALUES (?, vec_int8(?), ?, ?)
"""
else:
connection.executescript(
f"""
CREATE VIRTUAL TABLE vv USING vec0(
embedding bit[{dim}], atom_id text, holder_id text
);
"""
)
insert = """
INSERT INTO vv(rowid, embedding, atom_id, holder_id) VALUES (?, vec_bit(?), ?, ?)
"""
for start in range(0, count, 20_000):
stop = min(start + 20_000, count)
connection.executemany(
insert,
[(row, payload(row), str(row), f"h{row % 2}") for row in range(start, stop)],
)
return connection
def knn(connection: sqlite3.Connection, kind: str, query: Any, top: int, holder: str | None = None) -> list[int]:
"""Nearest rowids. Every statement here is written out in full: the only choice is which one."""
if holder is None:
if kind == "float":
rows = connection.execute(
"""
SELECT rowid FROM vv WHERE embedding MATCH ? AND k = ? ORDER BY distance
""",
(query, top),
).fetchall()
elif kind == "int8":
rows = connection.execute(
"""
SELECT rowid FROM vv WHERE embedding MATCH vec_int8(?) AND k = ? ORDER BY distance
""",
(query, top),
).fetchall()
else:
rows = connection.execute(
"""
SELECT rowid FROM vv WHERE embedding MATCH vec_bit(?) AND k = ? ORDER BY distance
""",
(query, top),
).fetchall()
elif kind == "float":
rows = connection.execute(
"""
SELECT rowid FROM vv WHERE embedding MATCH ? AND k = ? AND holder_id = ? ORDER BY distance
""",
(query, top, holder),
).fetchall()
elif kind == "int8":
rows = connection.execute(
"""
SELECT rowid FROM vv WHERE embedding MATCH vec_int8(?) AND k = ? AND holder_id = ? ORDER BY distance
""",
(query, top, holder),
).fetchall()
else:
rows = connection.execute(
"""
SELECT rowid FROM vv WHERE embedding MATCH vec_bit(?) AND k = ? AND holder_id = ? ORDER BY distance
""",
(query, top, holder),
).fetchall()
return [int(row[0]) for row in rows]
def measure(vectors: np.ndarray, probes: int) -> dict[str, Any]:
count, dim = vectors.shape
if probes > count:
raise SystemExit(f"--probes {probes} exceeds the {count} vectors in the corpus")
chosen = random.Random(SEED).sample(range(count), probes)
queries = vectors[chosen].copy()
cosine = vectors @ queries.T
threshold = -np.sort(-cosine, axis=0)[K - 1]
int8 = np.ascontiguousarray(np.clip(np.round(vectors * 127), -127, 127).astype(np.int8))
queries_int8 = np.ascontiguousarray(np.clip(np.round(queries * 127), -127, 127).astype(np.int8))
packed = np.packbits((vectors >= 0).astype(np.uint8), axis=1)
queries_packed = np.packbits((queries >= 0).astype(np.uint8), axis=1)
unpacked = np.unpackbits(packed, axis=1)
queries_unpacked = np.unpackbits(queries_packed, axis=1)
population = unpacked.sum(axis=1).astype(np.int32)
rows: list[dict[str, Any]] = []
tables: dict[str, sqlite3.Connection] = {}
for kind, payload, query_bytes in (
("float", lambda row: vectors[row].tobytes(), lambda probe: queries[probe].tobytes()),
("int8", lambda row: int8[row].tobytes(), lambda probe: queries_int8[probe].tobytes()),
("bit", lambda row: packed[row].tobytes(), lambda probe: queries_packed[probe].tobytes()),
):
tables[kind] = vec_table(kind, count, dim, payload)
rows.append(
run_arm(
f"sqlite-vec {kind}",
lambda probe, kind=kind, query_bytes=query_bytes: knn(tables[kind], kind, query_bytes(probe), TOP),
probes,
20,
cosine,
threshold,
)
)
rows.append(
run_arm(
f"sqlite-vec {kind} + holder filter",
lambda probe, kind=kind, query_bytes=query_bytes: knn(
tables[kind], kind, query_bytes(probe), TOP, holder="h0"
),
probes,
20,
cosine,
threshold,
)
)
rows.append(
run_arm(
"numpy float32 scan (the truth)",
lambda probe: list(np.argsort(-(vectors @ queries[probe]))[:TOP]),
probes,
10,
cosine,
threshold,
)
)
def hamming(probe: int) -> list[int]:
shared = unpacked @ queries_unpacked[probe]
return list(np.argsort(-(2 * shared - population))[:TOP])
rows.append(run_arm("numpy hamming (sign bit)", hamming, probes, 10, cosine, threshold))
for dims in TRUNCATIONS:
rows.append(
run_arm(
f"numpy float32 first {dims} dims",
lambda probe, dims=dims: list(np.argsort(-(vectors[:, :dims] @ queries[probe][:dims]))[:TOP]),
min(probes, 50),
5,
cosine,
threshold,
)
)
def rerank_outside(probe: int) -> list[int]:
coarse = knn(tables["bit"], "bit", queries_packed[probe].tobytes(), COARSE)
order = np.argsort(-(vectors[coarse] @ queries[probe]))[:TOP]
return [coarse[row] for row in order]
rows.append(run_arm(f"bit coarse@{COARSE} -> float32 rerank in numpy", rerank_outside, probes, 10, cosine, threshold))
return {
"commit": git_commit(),
"vectors": count,
"dimensions": dim,
"probes": probes,
"top_k": TOP,
"recall_note": (
"probes are drawn from the corpus itself, so these arms are compared with each other, "
"not quoted as retrieval quality; a hit is any returned atom whose true cosine is at "
f"least the {K}-th best, because byte-identical atoms make the id set ambiguous"
),
"bytes_per_vector": BYTES_PER_VECTOR,
"arms": rows,
"machine": machine_facts(),
}
def write_markdown(report: dict[str, Any], path: Path) -> None:
machine = report["machine"]
lines = [
f"# QUANT: what each vector encoding buys (commit {report['commit'][:9]})",
"",
(
f"- corpus: {report['vectors']} vectors, {report['dimensions']} dimensions, "
f"{report['probes']} probes, top {report['top_k']}"
),
(
f"- machine: {machine['system']} {machine['release']}, {machine['cpus']} cores, "
f"{machine['threads_per_query']} threads per query, load {machine['load_average_1_5_15']}"
),
f"- {report['recall_note']}",
"",
f"| arm | p50 ms | p95 ms | recall@{K} |",
"| --- | --- | --- | --- |",
]
for row in report["arms"]:
lines.append(
f"| {row['arm']} | {row['p50_ms']} | {row['p95_ms']} | {row[f'recall_at_{K}']} |"
)
lines += [
"",
"| encoding | bytes per vector |",
"| --- | --- |",
]
lines += [f"| {name} | {size} |" for name, size in report["bytes_per_vector"].items()]
lines += [
"",
(
"_Latency is wall time per query on the machine named above with the thread budget the "
"benchmark pins; a run on a loaded machine is not comparable._"
),
"",
]
path.write_text("\n".join(lines), encoding="utf-8")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--vectors", type=Path, default=DEFAULT_VECTORS)
parser.add_argument("--probes", type=int, default=PROBES)
parser.add_argument("--out", type=Path, default=Path("bench/reports/QUANT.md"))
parser.add_argument("--json-out", type=Path, default=Path("bench/reports/quant.json"))
args = parser.parse_args()
if args.probes <= 0:
raise SystemExit(f"--probes must be positive, got {args.probes}")
report = measure(load_matrix(args.vectors), args.probes)
args.json_out.parent.mkdir(parents=True, exist_ok=True)
args.json_out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
write_markdown(report, args.out)
for row in report["arms"]:
print(f"{row['arm']:42} p50 {row['p50_ms']:7.1f}ms recall@{K} {row[f'recall_at_{K}']}", flush=True)
if __name__ == "__main__":
main()
bench/scale_check.py
+26 -10
View File
@@ -14,33 +14,35 @@ import time
from pathlib import Path
from typing import Any
from bench.dense_scale import copy_store
from bench.query_set import load_queries, scale_probes
from bench.report import git_commit
from bench.run import latency
from bench.systems import machine_facts
DEFAULT_HOME = Path("bench/runs/baseline")
DEFAULT_SCRATCH = Path("bench/runs/scale")
MIN_ATOMS = 100_000
DEFAULT_PROBES = 300
def measure(home: Path, probes: int) -> dict[str, Any]:
"""Open the product store (migrating it in place if it is old) and time real searches."""
def measure(home: Path, scratch: Path, probes: int) -> dict[str, Any]:
"""Open a copy of the product store (migrating it if it is old) and time real searches."""
from ely_eye.config import Settings
from ely_eye.db import Database
from ely_eye.retrieval import RetrievalService
if not (home / "data" / "ely_eye.sqlite").is_file():
raise SystemExit(
f"{home / 'data' / 'ely_eye.sqlite'}: no product database, "
"build it with python -m bench.ingest_baseline"
)
settings = Settings(home=home)
copy_home, copy_seconds = copy_store(home, scratch)
settings = Settings(home=copy_home)
started = time.perf_counter()
database = Database(settings)
migration_seconds = time.perf_counter() - started
atoms = database.atom_count()
if atoms < MIN_ATOMS:
raise SystemExit(f"{database.path}: {atoms} atoms, this check needs at least {MIN_ATOMS}")
with database.connect() as connection:
schema = int(connection.execute("PRAGMA user_version").fetchone()[0])
print(f"{atoms} atoms copied in {copy_seconds:.1f}s, migrated in {migration_seconds:.1f}s", flush=True)
queries = scale_probes(load_queries(), probes)
service = RetrievalService(settings, database)
@@ -50,10 +52,15 @@ def measure(home: Path, probes: int) -> dict[str, Any]:
began = time.perf_counter()
service.search(query.text)
times.append(time.perf_counter() - began)
if len(times) % 100 == 0:
print(f"searched {len(times)}/{len(queries)}", flush=True)
return {
"commit": git_commit(),
"database": str(database.path),
"atoms": atoms,
"copy_seconds": copy_seconds,
"schema_version": schema,
"migration_seconds": migration_seconds,
"dense_embeddings_enabled": settings.dense_embeddings_enabled,
"probes": len(queries),
@@ -71,13 +78,18 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
[
"# SCALE: the product's own search at corpus size",
"",
f"- commit: `{report['commit']}`",
f"- database: {report['database']}",
(
f"- copied from the frozen baseline in {report['copy_seconds']:.1f}s and migrated "
f"in {report['migration_seconds']:.1f}s to schema {report['schema_version']}; "
"the baseline store itself is left untouched"
),
(
f"- atoms: {report['atoms']}, probes: {report['probes']} "
f"({', '.join(report['probe_modalities'])})"
),
f"- dense embeddings: {report['dense_embeddings_enabled']}",
f"- opening the store (migrations included): {report['migration_seconds']:.3f}s",
(
f"- p50 {timing['p50']:.4f}s · p95 {timing['p95']:.4f}s · max {timing['max']:.4f}s "
f"over {timing['queries']} queries"
@@ -88,6 +100,9 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
f"load {machine['load_average_1_5_15']}"
),
"",
"The baseline recorded p50 20.16s on this corpus because the old search pulled every",
"atom into Python and rebuilt a BM25 index per query.",
"",
(
"_Latency is wall time per search call on an idle machine with the thread budget "
"the benchmark pins; a run on a loaded machine is not comparable._"
@@ -102,6 +117,7 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--home", type=Path, default=DEFAULT_HOME)
parser.add_argument("--scratch", type=Path, default=DEFAULT_SCRATCH)
parser.add_argument("--probes", type=int, default=DEFAULT_PROBES)
parser.add_argument("--out", type=Path, default=Path("bench/reports/SCALE.md"))
parser.add_argument("--json-out", type=Path, default=Path("bench/reports/scale.json"))
@@ -109,7 +125,7 @@ def main() -> None:
if args.probes <= 0:
raise SystemExit(f"--probes must be positive, got {args.probes}")
report = measure(args.home, args.probes)
report = measure(args.home, args.scratch, args.probes)
args.json_out.parent.mkdir(parents=True, exist_ok=True)
args.json_out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
write_markdown(report, args.out)