"""Image and text encoding for cross-modal retrieval, over one small ONNX graph. Chinese-CLIP ViT-B/16 in fp16 (377 MB) keeps bilingual text and images in one vector space, so a screenshot can be found by a picture or by a sentence. The graph is the fused contrastive model: every call needs both towers' inputs, so the unused side is fed a placeholder row. """ from __future__ import annotations from pathlib import Path from typing import Any import numpy as np from PIL import Image IMAGE_SIZE = 224 TEXT_MAX_LENGTH = 52 IMAGE_MEAN = (0.48145466, 0.45782750, 0.40821073) IMAGE_STD = (0.26862954, 0.26130258, 0.27577711) DOWNLOAD_HINT = ( "huggingface-cli download Xenova/chinese-clip-vit-base-patch16 " "--include 'onnx/model_fp16.onnx' 'tokenizer*' 'config.json' --local-dir " ) def preprocess_image(image: Image.Image) -> np.ndarray: """Turn a PIL image into the (3, 224, 224) float32 tensor the graph expects. The reference processor resizes to a square and normalizes with CLIP constants; doing it here keeps torchvision out of the dependency list. """ rgb = image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.Resampling.BICUBIC) array = np.asarray(rgb, dtype=np.float32) / 255.0 mean = np.array(IMAGE_MEAN, dtype=np.float32) std = np.array(IMAGE_STD, dtype=np.float32) return ((array - mean) / std).transpose(2, 0, 1) class ClipEncoder: """One ONNX session, both towers, batched image encoding.""" def __init__(self, model_dir: Path) -> None: self.model_dir = Path(model_dir) self.onnx_path = self.model_dir / "onnx" / "model_fp16.onnx" if not self.onnx_path.exists(): raise SystemExit( f"{self.onnx_path} is not a Chinese-CLIP export; fetch the fp16 graph: {DOWNLOAD_HINT}" ) self._session: Any | None = None self._tokenizer: Any | None = None self._placeholder_text: dict[str, np.ndarray] | None = None self._placeholder_image: np.ndarray | None = None @property def session(self) -> Any: if self._session is None: import onnxruntime as ort self._session = ort.InferenceSession(str(self.onnx_path), providers=["CPUExecutionProvider"]) return self._session @property def tokenizer(self) -> Any: if self._tokenizer is None: from transformers import BertTokenizer # The export is BERT-style (vocab.txt, [CLS]=101); transformers 5 aliases the fast and slow tokenizers, so this is the name the CLIP docs mean. self._tokenizer = BertTokenizer.from_pretrained(str(self.model_dir)) return self._tokenizer def encode_images(self, images: list[Image.Image], batch_size: int = 8) -> np.ndarray: if not images: return np.zeros((0, 512), dtype=np.float32) rows: list[np.ndarray] = [] for start in range(0, len(images), batch_size): chunk = images[start : start + batch_size] pixels = np.stack([preprocess_image(image) for image in chunk]) rows.append(self._run("image_embeds", pixels, self._text_feed(1))[0]) return np.asarray(np.concatenate(rows), dtype=np.float32) def encode_texts(self, texts: list[str], batch_size: int = 16) -> np.ndarray: if not texts: return np.zeros((0, 512), dtype=np.float32) rows: list[np.ndarray] = [] for start in range(0, len(texts), batch_size): chunk = texts[start : start + batch_size] feed = self._text_feed(len(chunk), chunk) rows.append(self._run("text_embeds", self._image_feed(len(chunk)), feed)[0]) return np.asarray(np.concatenate(rows), dtype=np.float32) def _run(self, output: str, pixels: np.ndarray, text: dict[str, np.ndarray]) -> list[np.ndarray]: feed = {"pixel_values": pixels.astype(np.float32), **text} return self.session.run([output], feed) def _text_feed(self, batch: int, texts: list[str] | None = None) -> dict[str, np.ndarray]: if texts is None: if self._placeholder_text is None: self._placeholder_text = self._tokenize(["placeholder"]) return self._placeholder_text return self._tokenize(texts) def _image_feed(self, batch: int) -> np.ndarray: if self._placeholder_image is None: self._placeholder_image = np.zeros((1, 3, IMAGE_SIZE, IMAGE_SIZE), dtype=np.float32) return np.repeat(self._placeholder_image, batch, axis=0) def _tokenize(self, texts: list[str]) -> dict[str, np.ndarray]: encoded = self.tokenizer( texts, padding="max_length", truncation=True, max_length=TEXT_MAX_LENGTH, return_tensors="np", ) return { "input_ids": encoded["input_ids"].astype(np.int64), "attention_mask": encoded["attention_mask"].astype(np.int64), }