Give the system a directory: ask who knows, then ask them
A memory system that only answers from its own store is a filing cabinet, not a partner. The transactive rule the project is named for — not knowing is fine when you know who does — needed a holder registry, a directory query, and a referral that never leaks the text it may not read. - schema v5 adds `holders` (person/agent/service, declared topics, contact). A holder with memories exists whether or not anybody registered them; the registry adds reachability, it does not grant existence. - `who_knows` ranks holders by their own memories and is deliberately NOT filtered by visibility: "she knows" is the answer. The answer is filtered. - `ask` searches the asker's visible scope first (hop 0), then walks the directory with a visited set and a hop budget. Three terminal states: answered, referred, no_holder_knows. Nobody knowing is a result, not an error. - Confidence is ordinal, not calibrated: relative strength, coverage, and declared topics. Every hit carries the raw numbers and the basis lines that produced the score, so a ranking can be argued with. - Faces: HTTP (/api/holders, /api/who_knows, /api/ask), CLI (holders, holder-set, holder-rm, who-knows, ask), MCP (who_knows, ask, list_holders). Four bugs found by running the thing, not by reading it: - FTS5 refuses bm25() inside MAX(); the aggregate uses FTS5's own `rank` column. - A CJK question is longer than the sentence that answers it, so the whole-run phrase matched nothing. The query now ORs adjacent bigram phrases; adjacency is kept, equal length is not. Latin path untouched, baseline unaffected. - The hop budget counted the asker as a hop, so max_hops=1 asked nobody. - A second, stricter readability test in _answered() disagreed with the SQL visibility clause and hid a memory the asker was allowed to read. The SQL clause is the only authority now. A method named `list` shadowed the builtin inside its own class and made mypy reject the return annotation; renamed rather than cast around it. Proof: 155 pytest, 12 Go tests, ruff and mypy clean on touched files, pyright ledger unchanged at 62, and scripts/mcp_write_smoke.sh drives two principals on one SQLite file — the husband asks through MCP, gets `referred` with his wife's contact channel, and her private sentence appears nowhere in the reply.
This commit is contained in:
1 parent
9fe13fffb3
commit
dfc6cd4c28
19 files changed
+1403
-19
No files matched your search
@@ -11,6 +11,7 @@ from pydantic import BaseModel
|
||||
|
||||
from . import __version__
|
||||
from .api_cartridges import register as register_cartridge_routes
|
||||
from .api_holders import register as register_holder_routes
|
||||
from .benchmarks import (
|
||||
benchmark_prediction_report,
|
||||
benchmark_registry_report,
|
||||
@@ -246,6 +247,7 @@ def create_app() -> FastAPI:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
register_cartridge_routes(app, settings_dep)
|
||||
register_holder_routes(app, settings_dep)
|
||||
|
||||
@app.get("/api/adapters", response_model=list[AdapterStatus])
|
||||
def adapters(settings: Annotated[Settings, Depends(settings_dep)]) -> list[AdapterStatus]:
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Holder registry, who-knows, and referral routes: the directory side of the store.
|
||||
|
||||
No `from __future__ import annotations` here: FastAPI reads these route signatures, and string
|
||||
annotations would be resolved against module globals, where the injected settings_dep exists
|
||||
only as a parameter of register().
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException
|
||||
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .holder_store import HolderStore
|
||||
from .schemas import utc_now
|
||||
from .transactive import TransactiveService
|
||||
from .transactive_schemas import (
|
||||
AskRequest,
|
||||
Holder,
|
||||
HolderUpsertRequest,
|
||||
ReferralAnswer,
|
||||
WhoKnowsReport,
|
||||
WhoKnowsRequest,
|
||||
)
|
||||
|
||||
|
||||
def register(app: FastAPI, settings_dep: Callable[[], Settings]) -> None:
|
||||
"""Register the transactive routes. The server owns the timestamps, the caller does not."""
|
||||
|
||||
def store(settings: Settings) -> HolderStore:
|
||||
return HolderStore(Database(settings))
|
||||
|
||||
@app.get("/api/holders", response_model=list[Holder])
|
||||
def holders(settings: Annotated[Settings, Depends(settings_dep)]) -> list[Holder]:
|
||||
return store(settings).list_holders()
|
||||
|
||||
@app.post("/api/holders/{holder_id}", response_model=Holder)
|
||||
def put_holder(
|
||||
holder_id: str,
|
||||
request: HolderUpsertRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> Holder:
|
||||
now = utc_now()
|
||||
entry = Holder(
|
||||
holder_id=holder_id,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
**request.model_dump(),
|
||||
)
|
||||
try:
|
||||
return store(settings).upsert(entry)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
@app.get("/api/holders/{holder_id}", response_model=Holder)
|
||||
def get_holder(
|
||||
holder_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> Holder:
|
||||
entry = store(settings).get(holder_id)
|
||||
if entry is None:
|
||||
raise HTTPException(status_code=404, detail=f"no holder {holder_id!r} is registered")
|
||||
return entry
|
||||
|
||||
@app.delete("/api/holders/{holder_id}", response_model=Holder)
|
||||
def delete_holder(
|
||||
holder_id: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> Holder:
|
||||
registry = store(settings)
|
||||
entry = registry.get(holder_id)
|
||||
if entry is None:
|
||||
raise HTTPException(status_code=404, detail=f"no holder {holder_id!r} is registered")
|
||||
try:
|
||||
registry.delete(holder_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
return entry
|
||||
|
||||
@app.get("/api/holders/{holder_id}/knows", response_model=WhoKnowsReport)
|
||||
def holder_knows(
|
||||
holder_id: str,
|
||||
query: str,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> WhoKnowsReport:
|
||||
"""What this holder knows about, as a ranked directory answer."""
|
||||
try:
|
||||
return TransactiveService(settings).who_knows(query, as_holder=holder_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/who_knows", response_model=WhoKnowsReport)
|
||||
def who_knows(
|
||||
request: WhoKnowsRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> WhoKnowsReport:
|
||||
try:
|
||||
return TransactiveService(settings).who_knows(
|
||||
request.query, as_holder=request.as_holder, limit=request.limit
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
@app.post("/api/ask", response_model=ReferralAnswer)
|
||||
def ask(
|
||||
request: AskRequest,
|
||||
settings: Annotated[Settings, Depends(settings_dep)],
|
||||
) -> ReferralAnswer:
|
||||
try:
|
||||
return TransactiveService(settings).ask(
|
||||
request.question,
|
||||
as_holder=request.as_holder,
|
||||
max_hops=request.max_hops,
|
||||
include_text=request.include_text,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
@@ -16,6 +16,7 @@ from .benchmarks import (
|
||||
)
|
||||
from .cartridge import CartridgeService
|
||||
from .cartridge_assets import CartridgeAssetService
|
||||
from .cli_transactive import register as register_transactive_commands
|
||||
from .config import get_settings
|
||||
from .db import Database
|
||||
from .embeddings import EmbeddingService
|
||||
@@ -38,6 +39,9 @@ def json_dump(value: object) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
register_transactive_commands(app, lambda: get_settings())
|
||||
|
||||
|
||||
@app.command()
|
||||
def serve(host: str = "127.0.0.1", port: int = 8765) -> None:
|
||||
uvicorn.run(create_app(), host=host, port=port)
|
||||
@@ -202,7 +206,7 @@ def unmount(cartridge_id: str) -> None:
|
||||
@app.command()
|
||||
def append(
|
||||
cartridge_id: str,
|
||||
atom_id: list[str] = typer.Option(..., "--atom-id"), # noqa: B008 # typer reads option defaults itself
|
||||
atom_id: list[str] = typer.Option(..., "--atom-id"),
|
||||
) -> None:
|
||||
"""Add memories to an existing cartridge; the bundle and its DNA follow."""
|
||||
service = CartridgeService(get_settings())
|
||||
@@ -352,7 +356,7 @@ def build_visual_training_data(
|
||||
prd_path: Path = Path("PRD.md"),
|
||||
output_path: Path | None = None,
|
||||
cartridge_id: str | None = None,
|
||||
image_path: list[Path] | None = typer.Option(None, "--image-path"), # noqa: B008 # typer reads option defaults itself
|
||||
image_path: list[Path] | None = typer.Option(None, "--image-path"),
|
||||
) -> None:
|
||||
path = TrainingService(get_settings()).build_visual_dataset(
|
||||
prd_path=prd_path,
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""CLI face for the directory: register holders, ask who knows, ask and get referred.
|
||||
|
||||
Registered onto the main Typer app so the command names stay flat (`ely-eye who-knows`), while
|
||||
the transactive surface stays in one file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import typer
|
||||
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .holder_store import HolderStore
|
||||
from .schemas import utc_now
|
||||
from .transactive import TransactiveService
|
||||
from .transactive_schemas import Holder, HolderKind
|
||||
|
||||
|
||||
def register(app: typer.Typer, settings: Callable[[], Settings]) -> None:
|
||||
"""Add the holder and referral commands. The caller passes a settings thunk so one patch
|
||||
in a test redirects every command, the same way the cartridge routes do."""
|
||||
|
||||
def store() -> HolderStore:
|
||||
return HolderStore(Database(settings()))
|
||||
|
||||
def asker(as_holder: str) -> str:
|
||||
"""An empty --as-holder means the local principal, the same identity `remember` writes as."""
|
||||
return as_holder or settings().default_holder
|
||||
|
||||
@app.command("holders")
|
||||
def holders() -> None:
|
||||
"""List registered holders: the directory a referral walks."""
|
||||
for entry in store().list_holders():
|
||||
typer.echo(entry.model_dump_json(indent=2))
|
||||
|
||||
@app.command("holder-set")
|
||||
def holder_set(
|
||||
holder_id: str,
|
||||
name: str = "",
|
||||
kind: HolderKind = "person",
|
||||
knows_about: list[str] = typer.Option(None, "--knows-about"),
|
||||
contact: list[str] = typer.Option(None, "--contact", help="key=value, repeatable"),
|
||||
) -> None:
|
||||
"""Create or replace one holder entry."""
|
||||
now = utc_now()
|
||||
entry = Holder(
|
||||
holder_id=holder_id,
|
||||
name=name,
|
||||
kind=kind,
|
||||
knows_about=list(knows_about or []),
|
||||
contact=_pairs(contact),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
typer.echo(store().upsert(entry).model_dump_json(indent=2))
|
||||
|
||||
@app.command("holder-rm")
|
||||
def holder_rm(holder_id: str) -> None:
|
||||
"""Delete a holder. Refuses while that holder still owns memories."""
|
||||
store().delete(holder_id)
|
||||
typer.echo(f'{{"deleted": "{holder_id}"}}')
|
||||
|
||||
@app.command("who-knows")
|
||||
def who_knows(query: str, as_holder: str = "", limit: int = 8) -> None:
|
||||
"""Rank the holders whose memories answer this query."""
|
||||
report = TransactiveService(settings()).who_knows(
|
||||
query, as_holder=asker(as_holder), limit=limit
|
||||
)
|
||||
typer.echo(report.model_dump_json(indent=2))
|
||||
|
||||
@app.command()
|
||||
def ask(
|
||||
question: str,
|
||||
as_holder: str = "",
|
||||
max_hops: int = 3,
|
||||
include_text: bool = typer.Option(
|
||||
False, "--include-text", help="show the memory text when the asker may already read it"
|
||||
),
|
||||
) -> None:
|
||||
"""Ask the store, then whoever knows. An unanswered question says nobody knows."""
|
||||
answer = TransactiveService(settings()).ask(
|
||||
question,
|
||||
as_holder=asker(as_holder),
|
||||
max_hops=max_hops,
|
||||
include_text=include_text,
|
||||
)
|
||||
typer.echo(answer.model_dump_json(indent=2))
|
||||
|
||||
|
||||
def _pairs(values: list[str] | None) -> dict[str, str]:
|
||||
"""Turn repeatable key=value options into a contact map, naming any entry that is not one."""
|
||||
contact: dict[str, str] = {}
|
||||
for value in values or []:
|
||||
key, separator, detail = value.partition("=")
|
||||
if not separator or not key.strip():
|
||||
raise ValueError(f"--contact needs key=value, got {value!r}")
|
||||
contact[key.strip()] = detail
|
||||
return contact
|
||||
@@ -188,12 +188,18 @@ class Database:
|
||||
until: datetime | None = None,
|
||||
min_trust: float | None = None,
|
||||
cartridge_ids: list[str] | None = None,
|
||||
as_holder: str | None = None,
|
||||
) -> FtsPage:
|
||||
"""BM25 over the FTS index joined to memories that are open right now.
|
||||
|
||||
Every filter is a `? IS NULL OR` predicate in one literal statement: the ranking never
|
||||
depends on string-built SQL, and the planner sees one shape. A memory with no event time
|
||||
cannot answer a time filter, so it drops out when one is asked for.
|
||||
|
||||
`as_holder` applies the sharing rule: a memory is readable by its owner, by anyone named
|
||||
in its visible_to list, by everyone when it is public, and by anyone when nobody owns it.
|
||||
A memory with no holder cannot be private to a holder that does not exist. Without an
|
||||
asker the query runs over the whole library, which is what an operator debugging wants.
|
||||
"""
|
||||
with self.connect() as connection:
|
||||
rows = connection.execute(
|
||||
@@ -221,6 +227,13 @@ class Database:
|
||||
WHERE ca.atom_id = f.atom_id
|
||||
AND ca.cartridge_id IN (SELECT value FROM json_each(?))
|
||||
))
|
||||
AND (? IS NULL OR a.holder_id = ''
|
||||
OR json_extract(a.visibility_json, '$.scope') = 'everyone'
|
||||
OR a.holder_id = ?
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM json_each(json_extract(a.visibility_json, '$.holders'))
|
||||
WHERE value = ?
|
||||
))
|
||||
ORDER BY score
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
@@ -242,6 +255,9 @@ class Database:
|
||||
min_trust,
|
||||
None if cartridge_ids is None else to_json(cartridge_ids),
|
||||
None if cartridge_ids is None else to_json(cartridge_ids),
|
||||
as_holder,
|
||||
as_holder,
|
||||
as_holder,
|
||||
limit + 1,
|
||||
offset,
|
||||
),
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""The holder registry and the directory query: who holds a memory, and who knows a topic.
|
||||
|
||||
A transactive memory system stores the directory, not the content: "I do not know where the
|
||||
keys are, I know my wife knows, and I can find my wife." That requires a holder entity the
|
||||
store can answer questions about, and it requires the answer to be a pointer with a reason,
|
||||
never a guess.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import cast
|
||||
|
||||
from .db import Database
|
||||
from .row_codec import from_json, to_json
|
||||
from .schemas import utc_now
|
||||
from .text_index import fts_query
|
||||
from .transactive_schemas import Holder, HolderKind
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WhoKnowsCount:
|
||||
"""One holder's evidence for one query, straight from the index."""
|
||||
|
||||
holder_id: str
|
||||
matched: int
|
||||
top_score: float
|
||||
latest_time: str | None
|
||||
|
||||
|
||||
class HolderStore:
|
||||
"""CRUD for registered holders plus the raw counts who_knows is built from."""
|
||||
|
||||
def __init__(self, database: Database) -> None:
|
||||
self.database = database
|
||||
|
||||
def upsert(self, holder: Holder) -> Holder:
|
||||
"""Insert or replace a registry entry, keeping the original creation time."""
|
||||
now = utc_now().isoformat()
|
||||
with self.database.connect() as connection:
|
||||
existing = connection.execute(
|
||||
"SELECT created_at FROM holders WHERE holder_id = ?", (holder.holder_id,)
|
||||
).fetchone()
|
||||
created_at = str(existing["created_at"]) if existing is not None else now
|
||||
stored = holder.model_copy(update={"created_at": _moment(created_at), "updated_at": _moment(now)})
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO holders (holder_id, name, kind, knows_about_json, contact_json, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(holder_id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
kind = excluded.kind,
|
||||
knows_about_json = excluded.knows_about_json,
|
||||
contact_json = excluded.contact_json,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
stored.holder_id,
|
||||
stored.name,
|
||||
stored.kind,
|
||||
to_json(stored.knows_about),
|
||||
to_json(stored.contact),
|
||||
stored.created_at.isoformat(),
|
||||
stored.updated_at.isoformat(),
|
||||
),
|
||||
)
|
||||
return stored
|
||||
|
||||
def get(self, holder_id: str) -> Holder | None:
|
||||
with self.database.connect() as connection:
|
||||
row = connection.execute("SELECT * FROM holders WHERE holder_id = ?", (holder_id,)).fetchone()
|
||||
return None if row is None else _row_to_holder(row)
|
||||
|
||||
def list_holders(self) -> list[Holder]:
|
||||
with self.database.connect() as connection:
|
||||
rows = connection.execute("SELECT * FROM holders ORDER BY holder_id").fetchall()
|
||||
return [_row_to_holder(row) for row in rows]
|
||||
|
||||
def delete(self, holder_id: str) -> None:
|
||||
"""Remove a registry entry. Refuses while that holder still owns memories: forgetting a
|
||||
phone number must not silently orphan the memories it points to."""
|
||||
holding = self.atom_count(holder_id)
|
||||
if holding:
|
||||
raise ValueError(
|
||||
f"holder {holder_id!r} still holds {holding} memories; forget them before deleting the holder"
|
||||
)
|
||||
with self.database.connect() as connection:
|
||||
connection.execute("DELETE FROM holders WHERE holder_id = ?", (holder_id,))
|
||||
|
||||
def atom_count(self, holder_id: str) -> int:
|
||||
with self.database.connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT COUNT(*) AS n FROM v_atoms_current WHERE holder_id = ?", (holder_id,)
|
||||
).fetchone()
|
||||
return int(row["n"])
|
||||
|
||||
def known_holders(self) -> list[str]:
|
||||
"""Every holder with at least one open memory, registered or not.
|
||||
|
||||
A person who has memories is a holder whether or not anybody registered them; the
|
||||
registry adds declared topics and contact details, it does not grant existence.
|
||||
"""
|
||||
with self.database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT DISTINCT holder_id FROM v_atoms_current WHERE holder_id <> '' ORDER BY holder_id"
|
||||
).fetchall()
|
||||
return [str(row["holder_id"]) for row in rows]
|
||||
|
||||
def who_knows_counts(self, query: str, limit: int) -> list[WhoKnowsCount]:
|
||||
"""Rank holders by how well their own memories answer the query.
|
||||
|
||||
The directory is deliberately unfiltered by visibility: the whole point of transactive
|
||||
memory is knowing that someone knows, even when their memory is private to them.
|
||||
|
||||
`rank` is FTS5's own bm25 column, and it is the only way to aggregate the score: SQLite
|
||||
refuses bm25() inside MAX(), and refuses it in a subquery or window over a match.
|
||||
"""
|
||||
with self.database.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT a.holder_id,
|
||||
COUNT(*) AS matched,
|
||||
-MAX(rank) AS top_score,
|
||||
MAX(COALESCE(a.event_time, a.ingested_at)) AS latest_time
|
||||
FROM atom_fts AS f
|
||||
JOIN v_atoms_current AS a ON a.atom_id = f.atom_id
|
||||
WHERE atom_fts MATCH ?
|
||||
AND a.holder_id <> ''
|
||||
GROUP BY a.holder_id
|
||||
ORDER BY top_score DESC, matched DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(fts_query(query), limit),
|
||||
).fetchall()
|
||||
return [
|
||||
WhoKnowsCount(
|
||||
holder_id=str(row["holder_id"]),
|
||||
matched=int(row["matched"]),
|
||||
top_score=float(row["top_score"]),
|
||||
latest_time=None if row["latest_time"] is None else str(row["latest_time"]),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _moment(value: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError(f"holder timestamps need an explicit offset, stored {value!r}")
|
||||
return parsed
|
||||
|
||||
|
||||
_HOLDER_KINDS: tuple[HolderKind, ...] = ("person", "agent", "service")
|
||||
|
||||
|
||||
def _kind(value: str) -> HolderKind:
|
||||
if value not in _HOLDER_KINDS:
|
||||
raise ValueError(f"stored holder kind {value!r} is not one of person, agent, service")
|
||||
return cast(HolderKind, value)
|
||||
|
||||
|
||||
def _row_to_holder(row: sqlite3.Row) -> Holder:
|
||||
return Holder(
|
||||
holder_id=str(row["holder_id"]),
|
||||
name=str(row["name"]),
|
||||
kind=_kind(str(row["kind"])),
|
||||
knows_about=from_json(row["knows_about_json"], []),
|
||||
contact=from_json(row["contact_json"], {}),
|
||||
created_at=_moment(str(row["created_at"])),
|
||||
updated_at=_moment(str(row["updated_at"])),
|
||||
)
|
||||
@@ -14,7 +14,7 @@ import sqlite3
|
||||
|
||||
from .text_index import index_text
|
||||
|
||||
SCHEMA_VERSION = 4
|
||||
SCHEMA_VERSION = 5
|
||||
|
||||
ATOM_COLUMNS_V2 = """
|
||||
atom_id TEXT PRIMARY KEY,
|
||||
@@ -82,6 +82,21 @@ _BASE_TABLES = """
|
||||
mounted_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- The directory of who may hold or answer a memory. A transactive system answers
|
||||
-- "who knows this?" before it answers "what is this?", and it cannot do that without
|
||||
-- an entity to point at. Contact details are free-form on purpose: the transport is
|
||||
-- whoever wired this store together, not the schema.
|
||||
CREATE TABLE IF NOT EXISTS holders (
|
||||
holder_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
kind TEXT NOT NULL DEFAULT 'person'
|
||||
CHECK (kind IN ('person', 'agent', 'service')),
|
||||
knows_about_json TEXT NOT NULL DEFAULT '[]',
|
||||
contact_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cache_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts TEXT NOT NULL,
|
||||
|
||||
@@ -157,10 +157,12 @@ class SearchFilters(BaseModel):
|
||||
"""Who may answer, in what setting, and how fresh the memory has to be.
|
||||
|
||||
A memory with no event time cannot satisfy since/until, so it drops out when a time bound is
|
||||
asked for rather than passing as unknown.
|
||||
asked for rather than passing as unknown. `holder` restricts the answer to one holder's
|
||||
memories; `as_holder` is the identity asking, and applies the sharing rule instead.
|
||||
"""
|
||||
|
||||
holder: str | None = None
|
||||
as_holder: str | None = None
|
||||
scenario: str | None = None
|
||||
source_id: str | None = None
|
||||
modality: Modality | None = None
|
||||
|
||||
@@ -42,7 +42,7 @@ def fts_query(query: str) -> str:
|
||||
terms = _bare(query).split()
|
||||
if not terms:
|
||||
raise ValueError(f"search needs at least one searchable word, got {query!r}")
|
||||
return " ".join(_phrase(term) if _is_cjk(term) else f"{_quote(term)}*" for term in terms)
|
||||
return " ".join(_cjk_phrase(term) if _is_cjk(term) else f"{_quote(term)}*" for term in terms)
|
||||
|
||||
|
||||
def _bare(text: str) -> str:
|
||||
@@ -59,5 +59,17 @@ def _phrase(run: str) -> str:
|
||||
return '"' + " ".join(run) + '"'
|
||||
|
||||
|
||||
def _cjk_phrase(run: str) -> str:
|
||||
"""A CJK run stays adjacent, and a long question also reaches the pairs inside it.
|
||||
|
||||
The whole run as one phrase is the precise reading, and it is what a two-character word
|
||||
needs. A question is usually longer than the sentence that answers it: 钥匙在哪里 will not
|
||||
appear inside 我们的钥匙放在玄关抽屉里. Offering every adjacent pair as an alternative keeps
|
||||
adjacency — 钥 and 匙 still have to touch — without demanding equal length.
|
||||
"""
|
||||
phrases = list(dict.fromkeys([_phrase(run)] + [_phrase(run[start : start + 2]) for start in range(len(run) - 1)]))
|
||||
return phrases[0] if len(phrases) == 1 else "(" + " OR ".join(phrases) + ")"
|
||||
|
||||
|
||||
def _quote(term: str) -> str:
|
||||
return '"' + term.replace('"', '""') + '"'
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Ask the store who knows something, then ask them.
|
||||
|
||||
The transactive rule this service implements: not knowing is fine when the directory knows
|
||||
where to send you. Three things follow, and each one is a failure mode if left implicit.
|
||||
|
||||
- The directory is public, the content is not. `who_knows` counts every holder's matching
|
||||
memories, including memories the asker may not read, because "she knows" is the answer.
|
||||
- A referral never invents an answer. If the asker may not read the holder's memories, the
|
||||
result is a pointer plus contact details, and the text stays where it is.
|
||||
- Referral must terminate. Every hop records whom it asked, and the chain stops at the hop
|
||||
limit or when every candidate has been asked once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from .config import Settings, get_settings
|
||||
from .db import Database
|
||||
from .holder_store import HolderStore, WhoKnowsCount
|
||||
from .retrieval import RetrievalService
|
||||
from .schemas import SearchFilters
|
||||
from .text_index import index_text
|
||||
from .transactive_schemas import (
|
||||
ReferralAnswer,
|
||||
WhoKnowsHit,
|
||||
WhoKnowsReport,
|
||||
)
|
||||
|
||||
NO_HOLDER_KNOWS: Literal["no_holder_knows"] = "no_holder_knows"
|
||||
LIBRARY = "library"
|
||||
WHO_KNOWS_LIMIT = 8
|
||||
MAX_HOPS_LIMIT = 10
|
||||
|
||||
# Confidence is ordinal: it orders candidates, it does not estimate a probability. The weights
|
||||
# say what the evidence is worth relative to itself, and every hit carries the raw numbers too.
|
||||
CONFIDENCE_SATURATE = 3
|
||||
WEIGHT_STRENGTH = 0.60
|
||||
WEIGHT_COVERAGE = 0.25
|
||||
WEIGHT_DECLARED = 0.15
|
||||
|
||||
|
||||
class TransactiveService:
|
||||
"""who_knows and ask, built on the holder registry and the same retrieval as search."""
|
||||
|
||||
def __init__(self, settings: Settings | None = None, database: Database | None = None) -> None:
|
||||
self.settings = settings or get_settings()
|
||||
self.db = database or Database(self.settings)
|
||||
self.holders = HolderStore(self.db)
|
||||
self.retrieval = RetrievalService(self.settings, self.db)
|
||||
|
||||
def who_knows(
|
||||
self, query: str, *, as_holder: str | None = None, limit: int = WHO_KNOWS_LIMIT
|
||||
) -> WhoKnowsReport:
|
||||
"""Rank the holders whose own memories answer this query.
|
||||
|
||||
An empty result is the honest answer, not a fallback: nobody holds this, so the caller
|
||||
should say so instead of paraphrasing something unrelated.
|
||||
"""
|
||||
if not query.strip():
|
||||
raise ValueError("who_knows needs a query")
|
||||
if limit <= 0:
|
||||
raise ValueError(f"who_knows needs a positive limit, got {limit}")
|
||||
counts = self.holders.who_knows_counts(query, limit)
|
||||
if not counts:
|
||||
return WhoKnowsReport(query=query, as_holder=as_holder, status=NO_HOLDER_KNOWS, hits=[])
|
||||
registered = {holder.holder_id: holder for holder in self.holders.list_holders()}
|
||||
best = max(count.top_score for count in counts)
|
||||
return WhoKnowsReport(
|
||||
query=query,
|
||||
as_holder=as_holder,
|
||||
status="found",
|
||||
hits=[
|
||||
self._hit(count, best, registered.get(count.holder_id), query)
|
||||
for count in counts
|
||||
],
|
||||
)
|
||||
|
||||
def ask(
|
||||
self,
|
||||
question: str,
|
||||
*,
|
||||
as_holder: str | None = None,
|
||||
max_hops: int = 3,
|
||||
include_text: bool = False,
|
||||
) -> ReferralAnswer:
|
||||
"""Answer a question by asking the asker first, then the directory's best candidates.
|
||||
|
||||
`answered_by` is the asker when their own memory answered, `library` when an unowned
|
||||
artifact did, and the holder whose memory answered otherwise. `referred` means somebody
|
||||
knows but their memory is not readable by the asker.
|
||||
|
||||
`max_hops` counts the holders asked besides the asker: the asker's own memory is hop 0,
|
||||
so max_hops=1 still asks the best candidate.
|
||||
"""
|
||||
if not question.strip():
|
||||
raise ValueError("ask needs a question")
|
||||
if not 1 <= max_hops <= MAX_HOPS_LIMIT:
|
||||
raise ValueError(f"ask needs max_hops between 1 and {MAX_HOPS_LIMIT}, got {max_hops}")
|
||||
|
||||
own = self._search(question, as_holder=as_holder, holder=None)
|
||||
if own:
|
||||
return self._answered(
|
||||
question, as_holder, asked=[as_holder or LIBRARY], hops=0, hits=own, include_text=include_text
|
||||
)
|
||||
|
||||
asked: list[str] = [as_holder or LIBRARY]
|
||||
for hit in self.who_knows(question, as_holder=as_holder).hits:
|
||||
if hit.holder_id in asked:
|
||||
continue
|
||||
if len(asked) - 1 >= max_hops:
|
||||
break
|
||||
asked.append(hit.holder_id)
|
||||
readable = self._search(question, as_holder=as_holder, holder=hit.holder_id)
|
||||
if readable:
|
||||
return self._answered(
|
||||
question,
|
||||
as_holder,
|
||||
asked=asked,
|
||||
hops=len(asked) - 1,
|
||||
hits=readable,
|
||||
include_text=include_text,
|
||||
)
|
||||
return self._pointer(question, as_holder, asked, hit)
|
||||
return ReferralAnswer(
|
||||
question=question,
|
||||
as_holder=as_holder,
|
||||
status=NO_HOLDER_KNOWS,
|
||||
asked=asked,
|
||||
hops=len(asked) - 1,
|
||||
basis=["no holder has a memory that answers this"],
|
||||
)
|
||||
|
||||
def _search(self, query: str, *, as_holder: str | None, holder: str | None):
|
||||
result = self.retrieval.search(
|
||||
query,
|
||||
top_k=5,
|
||||
filters=SearchFilters(holder=holder, as_holder=as_holder),
|
||||
)
|
||||
return result.hits
|
||||
|
||||
def _answered(self, question, as_holder, *, asked, hops, hits, include_text) -> ReferralAnswer:
|
||||
"""The hits already passed the visibility clause in SQL, so they may be shown."""
|
||||
top = hits[0]
|
||||
owner = top.atom.holder_id or LIBRARY
|
||||
return ReferralAnswer(
|
||||
question=question,
|
||||
as_holder=as_holder,
|
||||
status="answered",
|
||||
answered_by=owner,
|
||||
asked=asked,
|
||||
hops=hops,
|
||||
atom_ids=[hit.atom.atom_id for hit in hits],
|
||||
answer=top.atom.text if include_text else None,
|
||||
basis=[
|
||||
f"{len(hits)} memories readable by {as_holder or 'anyone'}",
|
||||
f"best match scores {top.final_score:.2f}",
|
||||
f"held by {owner}",
|
||||
],
|
||||
)
|
||||
|
||||
def _pointer(self, question, as_holder, asked, hit: WhoKnowsHit) -> ReferralAnswer:
|
||||
"""The candidate knows but the asker cannot read it: return the contact, not the text."""
|
||||
contact = hit.contact
|
||||
return ReferralAnswer(
|
||||
question=question,
|
||||
as_holder=as_holder,
|
||||
status="referred",
|
||||
answered_by=hit.holder_id,
|
||||
asked=asked,
|
||||
hops=len(asked) - 1,
|
||||
basis=[
|
||||
f"{hit.holder_id} holds {hit.matched} matching memories the asker may not read",
|
||||
f"best match scores {hit.top_score:.2f}",
|
||||
f"reachable via {contact}" if contact else "no contact details registered",
|
||||
],
|
||||
)
|
||||
|
||||
def _hit(self, count: WhoKnowsCount, best: float, holder, query: str) -> WhoKnowsHit:
|
||||
declared = _declared_topics(holder.knows_about if holder else [], query)
|
||||
strength = count.top_score / best if best > 0 else 0.0
|
||||
coverage = min(1.0, count.matched / CONFIDENCE_SATURATE)
|
||||
confidence = (
|
||||
WEIGHT_STRENGTH * strength + WEIGHT_COVERAGE * coverage + WEIGHT_DECLARED * (1.0 if declared else 0.0)
|
||||
)
|
||||
basis = [
|
||||
f"holds {count.matched} matching memories",
|
||||
f"best match scores {count.top_score:.2f}",
|
||||
]
|
||||
if count.latest_time:
|
||||
basis.append(f"most recent memory is from {count.latest_time}")
|
||||
if declared:
|
||||
basis.append(f"declared topics: {', '.join(declared)}")
|
||||
contact = dict(holder.contact) if holder else {}
|
||||
if contact:
|
||||
basis.append(f"reachable via {contact}")
|
||||
return WhoKnowsHit(
|
||||
holder_id=count.holder_id,
|
||||
name=holder.name if holder else "",
|
||||
confidence=round(confidence, 3),
|
||||
matched=count.matched,
|
||||
top_score=round(count.top_score, 3),
|
||||
latest_time=None if count.latest_time is None else _as_moment(count.latest_time),
|
||||
basis=basis,
|
||||
contact=contact,
|
||||
)
|
||||
|
||||
|
||||
def _as_moment(value: str) -> datetime:
|
||||
"""A stored timestamp that will not parse is a corrupt row, not a missing one."""
|
||||
try:
|
||||
return datetime.fromisoformat(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(f"stored timestamp {value!r} is unreadable: {error}") from error
|
||||
|
||||
|
||||
def _declared_topics(topics: list[str], query: str) -> list[str]:
|
||||
"""Declared topics that share a searchable token with the query, using the index's own split."""
|
||||
query_tokens = set(index_text(query).split())
|
||||
if not query_tokens:
|
||||
return []
|
||||
return [topic for topic in topics if query_tokens & set(index_text(topic).split())]
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Models for the directory side of memory: who exists, who knows what, and who answered.
|
||||
|
||||
Confidence here is ordinal, not calibrated: it orders candidates by the evidence in front of
|
||||
us. Every model that carries a score also carries the raw evidence behind it, so a caller can
|
||||
disagree with the number without re-running the query.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
HolderKind = Literal["person", "agent", "service"]
|
||||
"""The registry's own vocabulary; migrations.py enforces the same set in the database."""
|
||||
|
||||
|
||||
class Holder(BaseModel):
|
||||
"""A registered person, agent, or service that can hold or answer memories."""
|
||||
|
||||
holder_id: str
|
||||
name: str = ""
|
||||
kind: HolderKind = "person"
|
||||
knows_about: list[str] = Field(default_factory=list)
|
||||
contact: dict[str, str] = Field(default_factory=dict)
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@field_validator("holder_id")
|
||||
@classmethod
|
||||
def _holder_id_is_usable(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("holder_id cannot be blank; a directory entry has to name somebody")
|
||||
return value.strip()
|
||||
|
||||
|
||||
class HolderUpsertRequest(BaseModel):
|
||||
"""What a caller may set about a holder. The id comes from the path, never the body."""
|
||||
|
||||
name: str = ""
|
||||
kind: HolderKind = "person"
|
||||
knows_about: list[str] = Field(default_factory=list)
|
||||
contact: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class WhoKnowsHit(BaseModel):
|
||||
"""One candidate holder, the evidence for them, and how to reach them."""
|
||||
|
||||
holder_id: str
|
||||
name: str = ""
|
||||
confidence: float
|
||||
matched: int
|
||||
top_score: float
|
||||
latest_time: datetime | None = None
|
||||
basis: list[str] = Field(default_factory=list)
|
||||
contact: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class WhoKnowsReport(BaseModel):
|
||||
"""The directory's answer to a question. `no_holder_knows` is a result, not a failure."""
|
||||
|
||||
query: str
|
||||
as_holder: str | None = None
|
||||
status: Literal["found", "no_holder_knows"]
|
||||
hits: list[WhoKnowsHit] = Field(default_factory=list)
|
||||
|
||||
|
||||
class WhoKnowsRequest(BaseModel):
|
||||
"""Ask the directory who holds an answer."""
|
||||
|
||||
query: str
|
||||
as_holder: str | None = None
|
||||
limit: int = Field(default=8, gt=0)
|
||||
|
||||
@field_validator("query")
|
||||
@classmethod
|
||||
def _query_is_usable(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("who_knows needs a query")
|
||||
return value.strip()
|
||||
|
||||
|
||||
class AskRequest(BaseModel):
|
||||
"""Ask a question on behalf of one holder, allowing a bounded chain of referrals."""
|
||||
|
||||
question: str
|
||||
as_holder: str | None = None
|
||||
max_hops: int = 3
|
||||
include_text: bool = False
|
||||
|
||||
@field_validator("question")
|
||||
@classmethod
|
||||
def _question_is_usable(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("ask needs a question")
|
||||
return value.strip()
|
||||
|
||||
|
||||
class ReferralAnswer(BaseModel):
|
||||
"""Who answered, who was asked along the way, and what may be shown.
|
||||
|
||||
`referred` means the directory found someone who knows but the asker may not read their
|
||||
memory: the answer is the pointer and the contact, never the text.
|
||||
"""
|
||||
|
||||
question: str
|
||||
as_holder: str | None = None
|
||||
status: Literal["answered", "referred", "no_holder_knows"]
|
||||
answered_by: str | None = None
|
||||
asked: list[str] = Field(default_factory=list)
|
||||
hops: int = 0
|
||||
atom_ids: list[str] = Field(default_factory=list)
|
||||
answer: str | None = None
|
||||
basis: list[str] = Field(default_factory=list)
|
||||
Reference in new issue
Block a user