ZacharyZhang-NY
fc0698c242
- bench/ingest_baseline.py drives the unmodified product IngestionService over all 50,789 corpus files in sharded SQLite DBs, then merges them. Thread and memory ceilings are set explicitly: ONNX Runtime defaulted to every core (95 threads/worker, load 490) and its CPU arena kept freed pages mapped, which OOM-killed workers. - bench/metrics.py: recall@k, MRR, nDCG@10, Wilson lower bound, chance baseline. No model, no product import, so the scorer cannot grade itself. - bench/systems.py + bench/run.py: the five ablation arms over the merged DB, latency percentiles, scale curve, and a top-10 equivalence check against the live RetrievalService. - bench/counterpart.py + bench/run_external.py: in-repo Mem0-style flat store and Zep-style temporal store as the external reference point. - bench/audit.py: enforces single-attempt scoring, frozen n, a chance row beside every score, and producer != verifier. It already flags the product's retry-until-target loop at backend/src/ely_eye/evals.py:2351. - backend/pyproject.toml: rapidocr-onnxruntime>=1.4.4 was unsatisfiable (PyPI maxes at 1.2.3); the successor is the renamed rapidocr package.
30 lines
634 B
TOML
30 lines
634 B
TOML
[tool.pyright]
|
|
venvPath = "."
|
|
venv = ".venv"
|
|
extraPaths = [".", "backend/src"]
|
|
pythonVersion = "3.12"
|
|
include = ["bench", "backend/src"]
|
|
|
|
[tool.mypy]
|
|
python_executable = ".venv/bin/python"
|
|
mypy_path = "."
|
|
python_version = "3.12"
|
|
files = ["bench", "backend/src"]
|
|
# bench/corpus is downloaded third-party data (git clones), not our source.
|
|
exclude = "bench/corpus/"
|
|
|
|
[[tool.mypy.overrides]]
|
|
module = "pyarrow.*"
|
|
ignore_missing_imports = true
|
|
|
|
[[tool.mypy.overrides]]
|
|
module = [
|
|
"rapidocr_onnxruntime.*",
|
|
"cv2.*",
|
|
"rank_bm25",
|
|
"sentence_transformers.*",
|
|
"onnxruntime.*",
|
|
"fitz.*",
|
|
]
|
|
ignore_missing_imports = true
|