ZacharyZhang-NY
03ca35763f
A screenshot with no OCR-able words used to be a memory nothing could reach. It now has a vector, and a question can be asked with it. - picture space: Xenova/chinese-clip-vit-base-patch16 fp16 ONNX, 361MB, sha256 0fad978faa89b85f…, CPU-runnable on the onnxruntime the repo already ships. Text 76ms, image 29ms/image at batch 8. The ONNX contract was measured, not assumed: it is the fused contrastive graph, so the unused tower is fed a placeholder row, and the tokenizer must be BertTokenizer — a CLIP tokenizer emits ids past the embedding table. - visual_index.py keeps picture vectors in their own family beside the text ones, keyed by atom id, and refuses to answer when the map and the array disagree. retrieval.py stays under 500 lines. - a search can be asked with a picture, and when the words come up thin the picture ranking joins the fusion — but only then, so a text query never pays for the encoder. - visual tokens are counted at 32x32 patches (migration v7): 21,325 image atoms went from 5,331 to 34,618 tokens, and the library from 29.9M to 84.2M. The 100M capsule fits this 50,278-file corpus and nothing much larger — that is what the claim actually means. - the vision-language describer is written and gated: with no snapshot an atom records "deferred: no vision-language model snapshot"; a broken model becomes a note on the atom instead of a dead ingest. - forget now voids the picture index and the copy this process holds, so a forgotten picture stops answering from RAM. Measured, six arms, ScreenSpot-v2 (554 icon crops, 408 whole screenshots), bench/reports/VISUAL.md: icon pool picture 0.208 words 0.175 chance 0.018 screenshot pool words 0.384 picture 0.215 screenshot to icon 0.108 — CLIP is a whole-image model, not a grounding model: it cannot point at one small icon inside a screen. within one screen MRR 0.908 (recall@10 is 1.000 there only because chance is 1.000 too, so MRR is the only honest number in that row). ScreenSpot-v2 instructions are all English, so this run does not test the bilingual claim and the report says so. Gates: 199 tests, ruff clean on 17 files, mypy clean on 16 (fresh cache), the repo's own pyright 0 errors on the 12 files touched.
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
"""What a picture costs the model that reads it.
|
|
|
|
The first version divided pixels by 224x224, which priced a 1280x720 frame at 18 tokens —
|
|
less than its caption. A vision model spends one token per patch, and Qwen3-VL's patch is
|
|
32x32, so the same frame is 920. A context budget built on the old number holds six times
|
|
less picture than it claims.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
import ely_eye.ingestion as ingestion_module
|
|
from ely_eye.config import Settings
|
|
from ely_eye.db import Database
|
|
from ely_eye.ingestion import IngestionService
|
|
from ely_eye.migrations import SCHEMA_VERSION, apply
|
|
from ely_eye.ocr import OcrResult
|
|
from ely_eye.storage import token_equivalent, visual_token_equivalent
|
|
|
|
FRAME = (1280, 720)
|
|
|
|
|
|
def _ingest(tmp_path: Path, settings: Settings) -> Database:
|
|
"""Run the real ingest path; each caller decides what its OCR says."""
|
|
database = Database(settings)
|
|
Image.new("RGB", FRAME, (10, 20, 30)).save(tmp_path / "frame.png")
|
|
IngestionService(settings=settings, database=database).ingest_path(tmp_path / "frame.png")
|
|
return database
|
|
|
|
|
|
def test_a_frame_is_priced_by_the_patches_a_vision_model_sees() -> None:
|
|
assert visual_token_equivalent(1280, 720) == 920
|
|
|
|
|
|
def test_a_small_tile_still_costs_more_than_a_single_token() -> None:
|
|
assert visual_token_equivalent(224, 224) == 49
|
|
|
|
|
|
def test_a_region_without_a_size_is_refused_rather_than_guessed() -> None:
|
|
for width, height in ((0, 720), (1280, -1)):
|
|
with pytest.raises(ValueError, match="needs a positive size"):
|
|
visual_token_equivalent(width, height)
|
|
|
|
|
|
def test_an_ingested_image_carries_its_text_and_its_patches(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
settings = Settings(home=tmp_path)
|
|
monkeypatch.setattr(ingestion_module, "read_text_from_image", lambda path: OcrResult("购物车", 0.9, []))
|
|
|
|
database = _ingest(tmp_path, settings)
|
|
atom = database.list_atoms()[0]
|
|
|
|
assert atom.token_equivalent == token_equivalent("购物车") + 920
|
|
|
|
|
|
def test_an_existing_store_is_repriced_by_the_migration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
settings = Settings(home=tmp_path)
|
|
monkeypatch.setattr(ingestion_module, "read_text_from_image", lambda path: OcrResult("", 0.0, []))
|
|
database = _ingest(tmp_path, settings)
|
|
note = tmp_path / "notes.md"
|
|
note.write_text("# 钥匙放在玄关抽屉里\n", encoding="utf-8")
|
|
IngestionService(settings=settings, database=database).ingest_path(note)
|
|
with database.connect() as connection:
|
|
connection.execute("UPDATE atoms SET token_equivalent = 18 WHERE modality = 'image'")
|
|
connection.execute("PRAGMA user_version = 6")
|
|
|
|
with database.connect() as connection:
|
|
apply(connection)
|
|
|
|
atoms = {atom.modality.value: atom for atom in database.list_atoms()}
|
|
assert atoms["image"].token_equivalent == 920, "the old 224x224 price must not survive the upgrade"
|
|
assert atoms["text"].token_equivalent == token_equivalent(note.read_text(encoding="utf-8"))
|
|
with database.connect() as connection:
|
|
assert int(connection.execute("PRAGMA user_version").fetchone()[0]) == SCHEMA_VERSION
|
|
|
|
|
|
def test_an_atom_without_a_layout_size_keeps_the_number_it_has(tmp_path: Path) -> None:
|
|
"""Nothing can be priced from a missing region, so the stored figure is left alone."""
|
|
settings = Settings(home=tmp_path)
|
|
database = Database(settings)
|
|
with database.connect() as connection:
|
|
apply(connection)
|
|
connection.execute("PRAGMA user_version = 6")
|
|
connection.execute(
|
|
"INSERT INTO sources (source_id, path, kind, mime, sha256, size_bytes, created_at, metadata_json)"
|
|
" VALUES ('s', '/tmp/x.png', 'image', 'image/png', 'h', 1, '2026-01-01T00:00:00+00:00', '{}')"
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO atoms (atom_id, source_id, modality, source, ingested_at, text, embedding_refs_json,"
|
|
" layout_json, relations_json, trust_json, metadata_json, token_equivalent, origin)"
|
|
" VALUES ('a1', 's', 'image', '/tmp/x.png', '2026-01-01T00:00:00+00:00', 'x', '[]', '{}', '[]', '{}', '{}', 18, 'artifact')"
|
|
)
|
|
|
|
with database.connect() as connection:
|
|
apply(connection)
|
|
assert int(connection.execute("SELECT token_equivalent FROM atoms WHERE atom_id = 'a1'").fetchone()[0]) == 18
|