Give spoken words a memory with an instant attached

An audio file used to produce no memory at all: _ingest_file returned an empty
batch for any suffix it did not know, with no log and no reason. It now raises
UnsupportedFile when you name the file and records a SkippedFile when it was a
directory walk, so a refusal is visible either way.

Speech is measured, not promised. Over 210.5 seconds of real Video-MME audio,
faster-whisper-tiny on CPU int8 transcribed at 51x realtime and reached word
error rate 0.242 against community subtitles nobody here wrote (a caption track
drops words a transcript keeps, so that is an upper bound). 66 speech atoms now
carry 550 word-level timestamps, 0 words out of order and 0 starting before
their own segment. The snapshot is 78.2 MB, under the 500 MB cap, and the report
is bench/reports/ASR.md.

Two library facts were measured rather than assumed. faster-whisper's own audio
loader is broken against PyAV 19 (av.open got an unexpected keyword
'metadata_errors') and it declares av with no version bound, so asr.decode_audio
decodes with PyAV directly and hands the model a float32 array instead of
pinning an old av with no cp314 wheel. And av.Error does not exist on PyAV 19 -
the real base is av.error.FFmpegError - so that is the only thing caught and my
own ValueErrors keep their messages.

Files over the 500-line rule were split rather than excused: ingestion.py (748)
into ingestion_common / ingestion_media / ingestion_repository / ingestion, and
cli.py (544) into cli_senses plus cli_common, following the cli_transactive
register pattern. MediaIngest declares _source_record abstract, so a half-wired
host fails at construction instead of returning None.

Verification: 207 tests pass; ruff clean on the 13 touched files; mypy clean on
them, and repo-wide mypy fell from 29 errors to 24 (5 fixed, none added);
pyright reports 0 errors on the touched files.
This commit is contained in:
ZacharyZhang-NY committed 2026-10-07 21:18:28 -04:00
1 parent 03ca35763f
commit 5289cb4b01
20 files changed
+1513 -611

No files matched your search

+6 -56
View File
@@ -16,11 +16,12 @@ from .benchmarks import (
)
from .cartridge import CartridgeService
from .cartridge_assets import CartridgeAssetService
from .cli_common import json_dump
from .cli_senses import register as register_sense_commands
from .cli_senses import senses_report
from .cli_transactive import register as register_transactive_commands
from .config import Settings, get_settings
from .config import get_settings
from .db import Database
from .describe import DEFERRED, VisionDescriber
from .embeddings import EmbeddingService
from .ingestion import IngestionService
from .memory import MemoryService
from .profiles import runtime_profile_report
@@ -32,18 +33,12 @@ from .retrieval import RetrievalService
from .runtime import QwenRuntime
from .schemas import MemoryOrigin, SearchFilters, TimePrecision, TrainingAdapterKind
from .training import TrainingService
from .visual_index import VisualIndex
app = typer.Typer(no_args_is_help=True)
def json_dump(value: object) -> str:
import json
return json.dumps(value, ensure_ascii=False, indent=2)
register_transactive_commands(app, lambda: get_settings())
register_sense_commands(app, lambda: get_settings())
@app.command()
@@ -51,29 +46,6 @@ def serve(host: str = "127.0.0.1", port: int = 8765) -> None:
uvicorn.run(create_app(), host=host, port=port)
def _senses(settings: Settings, db: Database) -> dict[str, dict[str, object]]:
"""What this machine can see, and what it has to pretend about.
A missing snapshot is reported, not hidden: an unrun capability is a plan, and the status
output is where an operator learns which claims about this install are real today.
"""
visual = VisualIndex(settings, db)
describer = VisionDescriber(settings)
index_present = visual.index_path.exists() and visual.map_path.exists()
return {
"picture_search": {
"model_dir": str(visual.model_dir() or ""),
"model_ready": visual.available(),
"index": "present" if index_present else "absent",
},
"picture_description": {
"model_dir": str(describer.model_dir() or ""),
"model_ready": describer.available(),
"note": "" if describer.available() else DEFERRED,
},
}
@app.command()
def status() -> None:
settings = get_settings()
@@ -91,7 +63,7 @@ def status() -> None:
),
"cartridges": db.cartridge_count(),
"data_home": str(settings.home),
"senses": _senses(settings, db),
"senses": senses_report(settings, db),
}
)
)
@@ -144,21 +116,6 @@ def search(
typer.echo(result.model_dump_json(indent=2))
@app.command()
def describe_image(path: Path) -> None:
"""Say what a picture shows; names the setup step when no vision model is installed."""
settings = get_settings()
typer.echo(json_dump({"image": str(path), "caption": VisionDescriber(settings).describe(path)}))
@app.command()
def index_visual() -> None:
"""Encode every image atom so a picture or a sentence can find it."""
settings = get_settings()
database = Database(settings)
typer.echo(json_dump(VisualIndex(settings, database).build()))
@app.command()
def remember(
text: str,
@@ -213,13 +170,6 @@ def forget(atom_id: str = "", source_id: str = "", reason: str = "") -> None:
typer.echo(report.model_dump_json(indent=2))
@app.command("build-dense-index")
def build_dense_index() -> None:
index = EmbeddingService(get_settings()).build_index()
dimensions = int(index.vectors.shape[1]) if index.vectors.size else 0
typer.echo(json_dump({"atoms": len(index.atom_ids), "dimensions": dimensions}))
@app.command("list-cartridges")
def list_cartridges() -> None:
for manifest in CartridgeService(get_settings()).list_manifests():