ZacharyZhang-NY
dc9cfb5a30
A memory used to have one state: present. A session could not be reloaded
without searching, nothing separated what a person said today from what the
store ingested a year ago, and two statements that disagree about a number
were both handed to the reader as if the store believed both.
Schema v10. Atoms gain session_id, importance, recall_count, last_recalled_at
and archived_at; two tables arrive beside them — consolidations (one watermark
per holder+scenario lane) and conflicts (a pair, the term they disagree on,
both values, and whether anyone has settled it). v_atoms_current now also
hides archived rows, and apply() drops the view before rebuilding it because
CREATE VIEW IF NOT EXISTS would keep the old body and ignore the archive.
Working memory is a read, not a store: `working <holder> <session>` returns
what that session recorded, in order, with no search involved. Consolidation
rolls a lane's new memories into one standing summary and moves the watermark
to the newest memory actually rolled up — not to the clock of the pass, which
made a second pass count the same memories twice. The archive rule is stated
in the command that runs it (recorded before N days, never returned by a
search, importance at or under a floor) rather than hidden in a decay formula
nobody measured; --dry-run names the victims first.
A conflict is recorded when a new memory repeats a label with a different
number in the same holder's same scenario. Retrieval drops the older side
while the conflict is open and both sides answer again once it is settled:
settling is not superseding, and a word-only disagreement is left alone
because a reworded sentence would fire on every paraphrase. Numbers are keyed
to both the memory's subject and the nearest preceding label, measured: the
two thermostat sentences share only their subject, so a nearest-word-only rule
found nothing.
Search now counts the memories it handed back, which is the only use estimate
the store has, and the archive rule reads it. The dense index applies the same
currency rules as the view, so an archived memory cannot stay findable through
one leg while the other hides it.
Two reads over HTTP (working, conflicts) and two tools over MCP
(working_set, memory_conflicts); curation stays with the operator because an
agent should not decide what its own past is worth. remember and supersede
carry a session id; importance does not cross MCP, because recall_count is
observed and importance is stated.
Gates: 366 passed; ruff clean; mypy 24 errors in 9 files, the recorded 9cb46c1
baseline, net 0; pyright 0 on the touched files; go build/vet/test and gofmt
clean; mcp_write_smoke PASS.
90 lines
3.9 KiB
Python
90 lines
3.9 KiB
Python
"""The leg ablation must stay an ablation: every leg on and off, and a report that says so."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from bench import leg_check, leg_report
|
|
|
|
|
|
def test_every_leg_appears_switched_both_ways() -> None:
|
|
dense = {config["dense"] for config in leg_check.CONFIGS}
|
|
visual = {config["visual"] for config in leg_check.CONFIGS}
|
|
assert dense == {True, False}, "the dense leg is only an ablation if both states are scored"
|
|
assert visual == {True, False}, "the picture leg is only an ablation if both states are scored"
|
|
pages = {int(config["top_k"]) for config in leg_check.CONFIGS}
|
|
assert leg_check.SHIPPED_TOP_K in pages, "the page size the product ships must be one of the rows"
|
|
|
|
|
|
def test_the_report_prints_every_arm_with_its_numbers(tmp_path: Path) -> None:
|
|
arm = {
|
|
"label": "words only",
|
|
"top_k": 10,
|
|
"dense": False,
|
|
"visual": False,
|
|
"recall_at_10": 0.832,
|
|
"mrr": 0.683,
|
|
"hit_at_10": 0.833,
|
|
"wilson_lower": 0.787,
|
|
"p50_seconds": 0.0637,
|
|
"p95_seconds": 0.13,
|
|
"median_distinct_sources": 4,
|
|
"by_modality": {"code": 0.953, "pdf": 0.849},
|
|
"modality_counts": {"code": 128, "pdf": 86},
|
|
"newest_gold_first": 0.44,
|
|
"memory": {
|
|
"queries": 150,
|
|
"recall_at_10": 0.965,
|
|
"mrr": 0.875,
|
|
"hit_at_10": 0.98,
|
|
"wilson_lower": 0.921,
|
|
"by_category": {"knowledge-update": 1.0, "temporal-reasoning": 0.947},
|
|
"category_counts": {"knowledge-update": 25, "temporal-reasoning": 25},
|
|
"newest_gold_first": 0.4,
|
|
"newest_gold_first_by_category": {"knowledge-update": 0.4, "temporal-reasoning": 0.526},
|
|
"newest_gold_first_counts": {"knowledge-update": 19, "temporal-reasoning": 21},
|
|
},
|
|
}
|
|
report: dict[str, Any] = {
|
|
"commit": "abcdef123456",
|
|
"schema_version": 8,
|
|
"store": "bench/runs/leg_check/data/ely_eye.sqlite",
|
|
"source_store": "bench/runs/dense_scale",
|
|
"dense_vectors": 153857,
|
|
"picture_index": {"path": "bench/runs/leg_check_index", "atoms": 53668, "dim": 512, "bytes": 109912192},
|
|
"encode_rate": {"pictures": 200, "seconds": 10.0, "per_second": 20.0},
|
|
"probes": 300,
|
|
"memory_queries": 150,
|
|
"memory_modality": "conversation",
|
|
"arms": [arm],
|
|
"machine": {
|
|
"system": "Linux",
|
|
"release": "6.18",
|
|
"cpus": 32,
|
|
"threads_per_query": 4,
|
|
"load_average_1_5_15": [1.0, 2.0, 3.0],
|
|
},
|
|
}
|
|
out = tmp_path / "LEGS.md"
|
|
leg_report.write_markdown(report, out)
|
|
text = out.read_text(encoding="utf-8")
|
|
assert "words only" in text
|
|
assert "0.832" in text and "63.7" in text, "recall and latency must sit in the same row"
|
|
assert "code" in text and "0.953" in text, "per-modality recall is where the picture legs show up"
|
|
assert (
|
|
"bench/runs/dense_scale" in text and "153,857 dense vectors" in text
|
|
), "a dense arm must name the store its vectors came from, not imply the frozen baseline"
|
|
assert (
|
|
"53,668 atoms at 512 dimensions" in text and "44.7" in text
|
|
), "the picture index needs its shape in the report, and its build cost derived from a measured rate"
|
|
assert (
|
|
"knowledge-update" in text and "1.000" in text and "0.40 (19)" in text
|
|
), "the two question types the contract names need their own recall and their own n"
|
|
assert (
|
|
"every answerable conversation query (150) measured whole" in text
|
|
), "a 15-probe sample cannot separate six question types, and the report must say which set the table used"
|
|
assert "| 0.44 |" in text, "the probe table needs its own newest-first column"
|
|
assert "unlabelled" not in text, "an unlabelled bucket would hide which set a number came from"
|
|
assert "abcdef123456" in text and "load_average" not in text
|