ZacharyZhang-NY
fdf9b956f5
Provider /models listings that return bare ids (OpenAI-style) get context windows, thinking levels, image support, and display names from models.dev: kigi-models owns the transform (parse_api_json — ONE field interpretation for the bundled snapshot AND runtime refreshes), enrich_wire_model fills gaps with wire values always winning and model availability strictly wire-truth. Spec rows gained models_dev_id + wire_serves_metadata; all three current platforms are wire-served, so this pipeline is provably inert for them (byte-identical catalogs, zero egress, zero ~/.kigi writes — adversarially verified). Shell side: enrichment_fetch with a 24h disk cache guarded by binary version + keep-set + future-stamp sanity (a registry change or downgrade refetches instead of serving a catalog missing new providers), refresh of https://models.dev/api.json filtered to registry ids, KIGI_MODELS_DEV_URL override with case/whitespace-tolerant kill switch, fallback chain fresh-cache > refresh > stale-cache > bundled (each step logged). The fast path returns an empty catalog without forcing the bundled parse. From the review: blast-radius-confined parsing (one drifted provider on models.dev warn-skips instead of failing the whole refresh), registry- coverage and field-coverage tests guarding script/parser drift, a path- injectable core with 8 state-machine tests (one of which caught a guard patch that had failed to apply), _meta provenance stamp in the snapshot, and models.dev (MIT) attribution in NOTICE. Snapshot: 29 providers, 1124 models, 246KB, regenerated by scripts/gen_enrichment_snapshot.py (pure filter, no transform).
78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Regenerate the bundled models.dev enrichment snapshot.
|
|
|
|
Usage:
|
|
curl -s https://models.dev/api.json -o /tmp/modelsdev-api.json
|
|
python3 scripts/gen_enrichment_snapshot.py /tmp/modelsdev-api.json
|
|
|
|
PURE FILTER, NO TRANSFORM: writes the raw models.dev per-provider objects
|
|
(only the providers kigi references) to
|
|
crates/codegen/kigi-models/enrichment_snapshot.json. All field
|
|
interpretation lives in ONE place — `kigi_models::enrichment::parse_api_json`
|
|
— which parses this snapshot and runtime refreshes identically.
|
|
|
|
Keep TARGETS in sync with the registry's `models_dev_id` values (a registry
|
|
test cross-checks coverage). `openrouter` is deliberately absent: its
|
|
/models wire serves context_length itself.
|
|
"""
|
|
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
TARGETS = [
|
|
"anthropic", "azure", "openai", "deepseek", "nvidia", "google",
|
|
"amazon-bedrock", "mistral", "groq", "cerebras", "cloudflare-ai-gateway",
|
|
"xai", "togetherai", "fireworks-ai", "opencode", "opencode-go",
|
|
"kimi-for-coding", "moonshotai", "moonshotai-cn", "minimax", "minimax-cn",
|
|
"alibaba-token-plan", "alibaba-token-plan-cn", "xiaomi",
|
|
"xiaomi-token-plan-cn", "zai-coding-plan", "zhipuai-coding-plan",
|
|
"github-copilot", "vercel",
|
|
]
|
|
|
|
# Per-model keys the Rust parser consumes; everything else is dropped to keep
|
|
# the bundled file lean (cost/date/experimental fields are dead weight).
|
|
MODEL_KEYS = ("name", "reasoning", "reasoning_options", "limit", "modalities",
|
|
"tool_call")
|
|
|
|
|
|
def main() -> int:
|
|
if len(sys.argv) != 2:
|
|
print(__doc__, file=sys.stderr)
|
|
return 2
|
|
src = json.load(open(sys.argv[1]))
|
|
missing = [pid for pid in TARGETS if pid not in src]
|
|
if missing:
|
|
# Fail fast: a target vanishing from models.dev needs a human look.
|
|
print(f"providers missing from api.json: {missing}", file=sys.stderr)
|
|
return 1
|
|
out = {}
|
|
# Provenance stamp (parse_api_json skips keys starting with "_").
|
|
out["_meta"] = {
|
|
"source": "https://models.dev/api.json (github.com/sst/models.dev, MIT)",
|
|
"note": "filtered snapshot for kigi model-metadata enrichment; "
|
|
"regenerate via scripts/gen_enrichment_snapshot.py",
|
|
}
|
|
out |= {
|
|
pid: {
|
|
"models": {
|
|
mid: {k: m[k] for k in MODEL_KEYS if k in m}
|
|
for mid, m in src[pid]["models"].items()
|
|
}
|
|
}
|
|
for pid in TARGETS
|
|
}
|
|
dst = pathlib.Path(__file__).resolve().parent.parent / (
|
|
"crates/codegen/kigi-models/enrichment_snapshot.json"
|
|
)
|
|
dst.write_text(json.dumps(out, separators=(",", ":"), sort_keys=True) + "\n")
|
|
providers = [k for k in out if not k.startswith("_")]
|
|
n_models = sum(len(out[p]["models"]) for p in providers)
|
|
print(f"wrote {dst}: {len(providers)} providers, {n_models} models, "
|
|
f"{dst.stat().st_size} bytes")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|