From 25c2b3aed269bc083540eb56e6baa371f37e84d2 Mon Sep 17 00:00:00 2001 From: markbus-ai <58405544+markbus-ai@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:47:44 -0300 Subject: [PATCH] perf(agents): cut codex usage collector memory with SQL filter and cache (#6780) * perf(agents): cut codex usage collector memory with SQL filter and cache The codex collector scanned every row of opencode.db (1.7 GB, 55k+ rows) with Python-side json.loads, peaking around 716 MB of RSS on every run -- including the panel's refreshLimits() call, which passed --limits-only that the collector silently ignored. Filter rows in SQL (LIKE gates + json_valid + json_extract authority, mirroring the old Python filter semantics) so giant blobs are never parsed, and cache the local stats scan in XDG_CACHE_HOME following the claude collector's pattern (atomic writes, flock, schemaVersion). --force rescans, --limits-only and normal mode reuse a fresh cache and fall back to a full scan when it is missing, stale, or corrupt. Measured: cold scan 716 MB -> 158 MB peak; warm --limits-only ~85 MB and ~1.4 s. Output record schema and values are unchanged for the same data (parity verified against the old filter, including malformed rows). * Scope the codex scan cache's 15-minute reuse to --limits-only A no-flag run is the widget's periodic refresh, and refreshIntervalSec is configurable down to 30 seconds; holding every mode to a 15-minute cache meant stats could lag far behind the interval the user asked for. Mirror the claude collector: normal runs reuse a scan for ~20 seconds purely to dedup concurrent collectors, and only --limits-only, which promises just fresh limits, may reuse a scan for up to 15 minutes. Co-Authored-By: Claude Fable 5 * Invalidate the codex scan cache across day boundaries The cached stats embed date-dependent fields (todayPrompts, todayTotalTokens, recentDays), but only the file's age was checked, so a cache written at 23:58 served yesterday's numbers as "today" for up to 15 minutes past midnight. Stamp the envelope with the scan's local date and treat any other date as a miss. Co-Authored-By: Claude Fable 5 * Reject codex scan caches with a future mtime A cache whose mtime is ahead of the clock has a negative age, which the freshness check accepted forever: setting the clock backwards froze the stats until real time caught up with the file. Require a non-negative age before trusting the cache. Co-Authored-By: Claude Fable 5 * Never cache an interrupted opencode scan A transient lock, schema migration, or corrupted database aborts the opencode scan mid-flight; the partial numbers still serve the current run, but persisting them let a single bad read suppress opencode usage for every cache reader until expiry. The claude collector already skips its opencode cache write on a database error; do the same here. Co-Authored-By: Claude Fable 5 * Make the json_valid guard order explicit in the opencode query The query relied on json_valid(data) evaluating before json_extract(), but SQLite does not promise that AND terms run left to right; a reordered plan would let json_extract raise on a malformed row and silently truncate the scan. Wrap each json_extract in a CASE so the guard is structural rather than positional. Co-Authored-By: Claude Fable 5 * Drop a claude-collector comment that is false for codex "These caches were world-readable before" was copied from the claude collector; codex had no caches before this one existed. Explain the chmod on its own terms. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: markbusking Co-authored-by: David Heinemeier Hansson Co-authored-by: Claude Fable 5 --- bin/omarchy-agent-usage-codex | 220 ++++++++- .../shell.d/agent-usage-codex-scanner-test.sh | 462 ++++++++++++++++++ 2 files changed, 658 insertions(+), 24 deletions(-) diff --git a/bin/omarchy-agent-usage-codex b/bin/omarchy-agent-usage-codex index 63229f0d..e0200535 100755 --- a/bin/omarchy-agent-usage-codex +++ b/bin/omarchy-agent-usage-codex @@ -11,12 +11,16 @@ panel only ever reads the JSON this prints. """ import argparse +import fcntl +import hashlib import json import os import select import shutil import sqlite3 import subprocess +import sys +import tempfile import time from datetime import datetime, timedelta, timezone from pathlib import Path @@ -25,6 +29,14 @@ AGENT_ID = "codex" AGENT_NAME = "Codex" AUTH_HELP = "Run `codex login` to authenticate." +# A scan this recent is only reused to dedup concurrent collector runs (the +# update command backgrounds one per agent while the panel refreshes on its +# own); every periodic widget refresh lands a real rescan, however low +# refreshIntervalSec is set. --limits-only promises only fresh limits, so it +# may reuse a scan for up to 15 minutes. +SCAN_REUSE_SECONDS = 20 +LIMITS_ONLY_REUSE_SECONDS = 900 + def local_day(value): if value is None: @@ -197,16 +209,47 @@ def scan_opencode_sessions(): # A subscription burned entirely through opencode leaves no native session # files, but opencode records per-message provider, model, and token usage # in its own database. Read-only: opencode may be writing right now. + # Returns whether the scan ran to completion: a scan cut short by a + # database error still contributes what it read, but must not be cached + # as if it were the whole story. db = Path(os.environ.get("XDG_DATA_HOME") or (Path.home() / ".local" / "share")) / "opencode" / "opencode.db" if not db.is_file(): - return + return True try: conn = sqlite3.connect(db.resolve().as_uri() + "?mode=ro", uri=True, timeout=2) except sqlite3.Error: - return + return False try: conn.execute("PRAGMA query_only = ON") - for session_id, raw in conn.execute("SELECT session_id, data FROM message"): + # OpenCode DBs grow huge (every historical message, JSON included), and + # Python-side json.loads of every row wasted ~600 MB of RSS on machines + # whose subscription never ran on OpenAI. The json_extract conditions are + # the authority for the rows that reach them: role == "assistant" and + # providerID == "openai", the same exact-match values the Python filter + # below checks. The guards in front are pure acceleration, not a perfect + # proxy for the Python filter: + # - The LIKE gates skip rows whose JSON cannot contain the two + # key/value pairs, avoiding the JSON parse of tens-of-MB blobs. They + # can differ from json.loads on duplicate keys (Python keeps the + # last, SQLite json_extract keeps the first) and ASCII-escaped + # values (\"ass\\u0069stant\" decodes for Python but not for LIKE), + # so a row the authority would accept can be gated out. Both cases + # are vanishingly rare in real opencode data. + # - json_valid guards the parse itself: json_extract() RAISES on + # malformed JSON instead of returning NULL, and one such row would + # otherwise abort the whole scan. SQLite does not promise that AND + # terms evaluate left to right, so the guard is a CASE around each + # json_extract rather than a separate AND term. Rows that are not + # well-formed JSON are skipped here; the per-row try/except below + # stays as the final safety net for rows that pass the SQL filter + # but fail json.loads. + for session_id, raw in conn.execute( + "SELECT session_id, data FROM message" + " WHERE data LIKE '%\"role\"%:%\"assistant\"%'" + " AND data LIKE '%\"providerID\"%:%\"openai\"%'" + " AND CASE WHEN json_valid(data) THEN json_extract(data, '$.role') END = 'assistant'" + " AND CASE WHEN json_valid(data) THEN json_extract(data, '$.providerID') END = 'openai'" + ): # One malformed row must not abort the scan, so every shape assumption # lives inside the try. try: @@ -232,9 +275,12 @@ def scan_opencode_sessions(): continue add_usage(day, "opencode:" + str(session_id), model, input_tokens, output_tokens, cache_read, cache_write) except sqlite3.Error: - pass + # Transient lock, schema migration, corruption: the numbers stop here, + # incomplete. + return False finally: conn.close() + return True def scan_native_codex_sessions(): @@ -291,6 +337,143 @@ def scan_native_codex_sessions(): continue +def cache_root(): + root = Path(os.environ.get("XDG_CACHE_HOME") or (Path.home() / ".cache")) / "omarchy" / "agent-usage" + root.mkdir(parents=True, exist_ok=True) + return root + + +def scan_cache_paths(): + codex_home = Path(os.environ.get("CODEX_HOME") or (Path.home() / ".codex")) + db = Path(os.environ.get("XDG_DATA_HOME") or (Path.home() / ".local" / "share")) / "opencode" / "opencode.db" + # The digest covers every data path the scan reads: the codex session + # roots, the opencode DB, and (via Path.home()) the pi/omp session roots. + digest = hashlib.sha1((str(Path.home()) + "\n" + str(codex_home) + "\n" + str(db)).encode("utf-8")).hexdigest()[:16] + root = cache_root() + return root / f"codex-scan-{digest}.json", root / f"codex-scan-{digest}.lock" + + +def read_fresh_json(path, max_age_seconds): + if max_age_seconds <= 0 or not path.exists(): + return None + try: + # A negative age means the mtime is in the future: the clock moved + # backwards since the write, so the cache's freshness cannot be trusted. + age = time.time() - path.stat().st_mtime + if 0 <= age <= max_age_seconds: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None + return None + + +def write_json(path, payload): + # A temp name unique to this writer, not derived from the target: several + # collectors can run at once (the update command backgrounds one per agent, + # the panel refreshes on its own), and a shared temp path means the second + # replace finds the first one's file already moved away. + handle_fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=path.name + ".", suffix=".tmp") + tmp = Path(tmp_name) + try: + with os.fdopen(handle_fd, "w", encoding="utf-8") as handle: + handle.write(json.dumps(payload, separators=(",", ":")) + "\n") + # mkstemp opens at 0600; nothing in the cache is sensitive, so open it + # up to the usual 0644. + tmp.chmod(0o644) + tmp.replace(path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +# The cache payload is a versioned envelope around the local-stats dict, so a +# corrupted or foreign-shaped file is a cache miss (rescan + rewrite) instead +# of a crash or a garbage record. +def read_cached_stats(cache_file, max_age_seconds): + cached = read_fresh_json(cache_file, max_age_seconds) + if not isinstance(cached, dict) or cached.get("schemaVersion") != 1: + return None + # today* fields only mean "today" on the day they were scanned. A cache + # from another local date (midnight passed, or the clock moved) is a miss, + # not merely old, whatever its mtime says. + if cached.get("scanDate") != today: + return None + stats = cached.get("stats") + if not isinstance(stats, dict): + return None + if not all(key in stats for key in ("todayPrompts", "todayTotalTokens", "recentDays", "activeDates", "modelUsage")): + return None + return stats + + +def write_cached_stats(cache_file, stats): + try: + write_json(cache_file, {"schemaVersion": 1, "scanDate": today, "stats": stats}) + except Exception as exc: + print(f"omarchy-agent-usage-codex: could not write usage cache ({exc})", file=sys.stderr) + + +def local_stats(): + """Snapshot the aggregated local usage into the record's stats dict.""" + return { + "todayPrompts": today_prompts, + "todaySessions": len(today_sessions), + "todayTotalTokens": today_total_tokens, + "todayTokensByModel": today_tokens_by_model, + "recentDays": [recent[day] for day in recent_dates], + "totalPrompts": total_prompts, + "totalSessions": len(total_sessions), + # Days with any recorded usage, for the all-time "N days" summary. The + # dates travel too: merging snapshots from several machines needs their + # union, which a count alone cannot give. + "activeDays": len(active_days), + "activeDates": sorted(active_days), + "modelUsage": model_usage, + } + + +def run_local_scans(): + scan_pi_sessions() + scan_native_codex_sessions() + complete = scan_opencode_sessions() + return local_stats(), complete + + +def cached_local_stats(max_age): + """Local stats, with the cache as a pure optimization. + + The cache must never take the collector down: any cache-layer failure + (unwritable cache root, lock errors, disk full) degrades to a direct scan + and a warning on stderr. The JSON record is the contract; the cache is not. + """ + try: + return _cached_local_stats(max_age) + except Exception as exc: + print(f"omarchy-agent-usage-codex: cache unavailable ({exc}); scanning directly", file=sys.stderr) + stats, _ = run_local_scans() + return stats + + +def _cached_local_stats(max_age): + cache_file, lock_file = scan_cache_paths() + + cached = read_cached_stats(cache_file, max_age) + if cached is not None: + return cached + + with lock_file.open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + cached = read_cached_stats(cache_file, max_age) + if cached is not None: + return cached + stats, complete = run_local_scans() + # An interrupted scan still serves this run, but caching it would + # suppress the missing usage for every reader until the cache expires. + if complete: + write_cached_stats(cache_file, stats) + return stats + + def rpc_request(proc, request_id, method, params=None, timeout=8): payload = {"id": request_id, "method": method, "params": params or {}} proc.stdin.write(json.dumps(payload) + "\n") @@ -390,16 +573,17 @@ def fetch_codex_rpc(): def main(): parser = argparse.ArgumentParser() - # Local stats and limits come from the same cheap scan, so there is no - # cache to force past and no faster limits-only path. The flags exist so - # every collector accepts the same invocation. + # --force rescans everything and rewrites the cache. --limits-only is kept + # for CLI compatibility with the panel's refreshLimits() call: only the + # limits probe must be fresh, so it may reuse a scan for far longer than a + # normal run, whose short window exists purely to dedup concurrent + # collector runs. parser.add_argument("--force", action="store_true") parser.add_argument("--limits-only", action="store_true") - parser.parse_args() + args = parser.parse_args() - scan_pi_sessions() - scan_native_codex_sessions() - scan_opencode_sessions() + max_age = 0 if args.force else (LIMITS_ONLY_REUSE_SECONDS if args.limits_only else SCAN_REUSE_SECONDS) + stats = cached_local_stats(max_age) rpc = fetch_codex_rpc() record = { @@ -409,20 +593,8 @@ def main(): "updatedAt": datetime.now(timezone.utc).isoformat(), "ready": True, "hasLocalStats": True, - "todayPrompts": today_prompts, - "todaySessions": len(today_sessions), - "todayTotalTokens": today_total_tokens, - "todayTokensByModel": today_tokens_by_model, - "recentDays": [recent[day] for day in recent_dates], - "totalPrompts": total_prompts, - "totalSessions": len(total_sessions), - # Days with any recorded usage, for the all-time "N days" summary. The - # dates travel too: merging snapshots from several machines needs their - # union, which a count alone cannot give. - "activeDays": len(active_days), - "activeDates": sorted(active_days), - "modelUsage": model_usage, } + record.update(stats) record.update(rpc) print(json.dumps(record, separators=(",", ":"))) diff --git a/test/shell.d/agent-usage-codex-scanner-test.sh b/test/shell.d/agent-usage-codex-scanner-test.sh index 4b31767e..f3f9aa1e 100644 --- a/test/shell.d/agent-usage-codex-scanner-test.sh +++ b/test/shell.d/agent-usage-codex-scanner-test.sh @@ -128,3 +128,465 @@ pass "Codex collector counts OpenAI usage, reasoning included, from opencode ses [[ $(jq -c '.modelUsage' <<<"$result") == '{"gpt-5.2-codex":{"inputTokens":80,"outputTokens":45,"cacheReadInputTokens":30,"cacheCreationInputTokens":0}}' ]] || fail "Codex collector ignores prefix-colliding providers, user messages, and malformed rows" "$result" pass "Codex collector ignores prefix-colliding providers, user messages, and malformed rows" + +# A warm cache makes --limits-only cheap: local stats come from the last scan +# instead of another walk over the opencode database, and --force bypasses it. +CACHE_HOME=$(mktemp -d) +trap 'rm -rf "$TEST_HOME" "$PI_HOME" "$OPENCODE_HOME" "$CACHE_HOME" "$FRESH_HOME"' EXIT +mkdir -p "$CACHE_HOME/bin" +cp "$TEST_HOME/bin/codex" "$CACHE_HOME/bin/codex" + +python3 - "$CACHE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +db.parent.mkdir(parents=True, exist_ok=True) +conn = sqlite3.connect(db) +conn.execute("CREATE TABLE message (id text PRIMARY KEY, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL)") +now_ms = int(time.time() * 1000) + +def message(id, provider, model, role="assistant", input=0, output=0, reasoning=0, read=0, write=0): + return (id, "ses_1", now_ms, now_ms, json.dumps({ + "role": role, + "providerID": provider, + "modelID": model, + "tokens": {"input": input, "output": output, "reasoning": reasoning, "cache": {"read": read, "write": write}}, + "time": {"created": now_ms}, + })) + +conn.executemany("INSERT INTO message VALUES (?, ?, ?, ?, ?)", [ + message("c_1", "openai", "gpt-5.2-codex", input=5), +]) +conn.commit() +conn.close() +PY + +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "5" ]] || + fail "Codex collector writes a fresh local-stats cache on first scan" "$result" +cache_file=$(ls "$CACHE_HOME/.cache/omarchy/agent-usage/"/codex-scan-*.json 2>/dev/null | head -n 1) +[[ -n $cache_file && -s $cache_file ]] || + fail "Codex collector leaves a cache file behind" "$result" +[[ $(stat -c %a "$cache_file") == "644" ]] || + fail "Codex collector keeps cache files readable" "$result" +[[ $(jq -r '.schemaVersion' "$cache_file") == "1" && $(jq -r '.stats.todayTotalTokens' "$cache_file") == "5" ]] || + fail "Codex collector writes a versioned cache envelope" "$result" +pass "Codex collector writes a local-stats cache on first scan" + +# A corrupt-but-parseable cache (wrong shape) is a cache miss: rescan and +# rewrite instead of emitting a garbage record. +printf '[]' >"$cache_file" +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "5" ]] || + fail "Codex collector recovers from a corrupt cache file" "$result" +[[ $(jq -r '.schemaVersion' "$cache_file") == "1" ]] || + fail "Codex collector rewrites the cache after a corrupt read" "$result" +pass "Codex collector recovers from a corrupt cache file" + +# A new opencode message changes what a scan would find; a --limits-only run +# must reuse the cached stats instead of rescanning. +python3 - "$CACHE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +conn = sqlite3.connect(db) +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "c_2", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 10, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY + +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "5" ]] || + fail "Codex collector --limits-only reuses cached local stats" "$result" +[[ $(jq -c '.modelUsage' <<<"$result") == '{"gpt-5.2-codex":{"inputTokens":5,"outputTokens":0,"cacheReadInputTokens":0,"cacheCreationInputTokens":0}}' ]] || + fail "Codex collector --limits-only emits a complete record from cache" "$result" +pass "Codex collector --limits-only reuses cached local stats" + +# --force must ignore the cache and pick up the new message. +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --force) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "15" ]] || + fail "Codex collector --force rescans past the cache" "$result" +pass "Codex collector --force rescans past the cache" + +# The forced scan refreshed the cache, so a following --limits-only sees it. +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "15" ]] || + fail "Codex collector --limits-only sees a refreshed cache after --force" "$result" +pass "Codex collector --limits-only sees a refreshed cache after --force" + +# An expired cache makes --limits-only rescan too: stale today* stats must +# never be served under a fresh updatedAt. +python3 - "$CACHE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +conn = sqlite3.connect(db) +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "c_3", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 10, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY +touch -d "2 hours ago" "$cache_file" +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "25" ]] || + fail "Codex collector --limits-only rescans when the cache is stale" "$result" +pass "Codex collector --limits-only rescans when the cache is stale" + +# The 15-minute reuse window belongs to --limits-only alone. A no-flag run +# (the widget's periodic refresh) reuses a scan only while it is young enough +# to be a concurrent collector run; past that it rescans, so stats stay as +# fresh as refreshIntervalSec, however low the user sets it. +python3 - "$CACHE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +conn = sqlite3.connect(db) +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "c_4", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 10, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY + +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "25" ]] || + fail "Codex collector no-flag reuses a seconds-old cache" "$result" + +# 30 seconds is the lowest refreshIntervalSec the widget supports, so a +# cache that old must already be past the no-flag reuse window. +touch -d "30 seconds ago" "$cache_file" +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "35" ]] || + fail "Codex collector no-flag rescans past the concurrent-run window" "$result" +pass "Codex collector no-flag mode rescans instead of serving a stale cache" + +# The same age from the other side: a cache far past the no-flag window but +# well inside 15 minutes is still good enough for --limits-only. +python3 - "$CACHE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +conn = sqlite3.connect(db) +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "c_5", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 10, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY +touch -d "10 minutes ago" "$cache_file" +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "35" ]] || + fail "Codex collector --limits-only reuses a scan the no-flag mode would refresh" "$result" +pass "Codex collector --limits-only reuses a scan the no-flag mode would refresh" + +# A cache written on another local date holds another day's today* stats even +# under a fresh mtime (midnight passed, or the clock moved backwards): the +# envelope's scanDate must turn it into a miss. +jq -c '.scanDate = "1999-01-01"' "$cache_file" >"$cache_file.tmp" && mv "$cache_file.tmp" "$cache_file" +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "45" ]] || + fail "Codex collector treats a cache from another day as a miss" "$result" +[[ $(jq -r '.scanDate' "$cache_file") == "$(date +%Y-%m-%d)" ]] || + fail "Codex collector stamps the rewritten cache with the scan date" "$result" +pass "Codex collector treats a cache from another day as a miss" + +# A cache stamped in the future (the clock was set backwards after the write) +# has no trustworthy age: it must be a miss, not fresh until the clock +# catches up. +python3 - "$CACHE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +conn = sqlite3.connect(db) +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "c_6", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 10, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY +touch -d "@$(( $(date +%s) + 3600 ))" "$cache_file" +result=$(HOME="$CACHE_HOME" CODEX_HOME="$CACHE_HOME/.codex" XDG_CACHE_HOME="$CACHE_HOME/.cache" XDG_DATA_HOME="$CACHE_HOME/.local/share" \ + PATH="$CACHE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "55" ]] || + fail "Codex collector treats a future-dated cache as a miss" "$result" +pass "Codex collector treats a future-dated cache as a miss" + +# First --limits-only on a machine with no cache falls back to a full scan. +FRESH_HOME=$(mktemp -d) +trap 'rm -rf "$TEST_HOME" "$PI_HOME" "$OPENCODE_HOME" "$CACHE_HOME" "$FRESH_HOME"' EXIT +mkdir -p "$FRESH_HOME/bin" +cp "$TEST_HOME/bin/codex" "$FRESH_HOME/bin/codex" + +python3 - "$FRESH_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +db.parent.mkdir(parents=True, exist_ok=True) +conn = sqlite3.connect(db) +conn.execute("CREATE TABLE message (id text PRIMARY KEY, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL)") +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "f_1", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 7, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY + +result=$(HOME="$FRESH_HOME" CODEX_HOME="$FRESH_HOME/.codex" XDG_CACHE_HOME="$FRESH_HOME/.cache" XDG_DATA_HOME="$FRESH_HOME/.local/share" \ + PATH="$FRESH_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "7" ]] || + fail "Codex collector --limits-only falls back to a full scan without a cache" "$result" +pass "Codex collector --limits-only falls back to a full scan without a cache" + +# A malformed opencode row must not abort the scan: json_valid() guards the +# parse, so the good rows are still counted. Real opencode data also stores +# compact JSON, so one row is serialized compactly here on purpose. +MALFORMED_HOME=$(mktemp -d) +trap 'rm -rf "$TEST_HOME" "$PI_HOME" "$OPENCODE_HOME" "$CACHE_HOME" "$FRESH_HOME" "$MALFORMED_HOME"' EXIT +mkdir -p "$MALFORMED_HOME/bin" +cp "$TEST_HOME/bin/codex" "$MALFORMED_HOME/bin/codex" + +python3 - "$MALFORMED_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +db.parent.mkdir(parents=True, exist_ok=True) +conn = sqlite3.connect(db) +conn.execute("CREATE TABLE message (id text PRIMARY KEY, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL)") +now_ms = int(time.time() * 1000) + +def message(id, input=0, compact=False): + payload = { + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": input, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + } + if compact: + return (id, "ses_1", now_ms, now_ms, json.dumps(payload, separators=(",", ":"))) + return (id, "ses_1", now_ms, now_ms, json.dumps(payload)) + +conn.executemany("INSERT INTO message VALUES (?, ?, ?, ?, ?)", [ + message("mm_1", input=5, compact=True), + message("mm_2", input=7), +]) +# Valid JSON followed by trailing garbage: without json_valid() this row +# makes json_extract() raise and aborts the whole scan. +good = json.dumps({"role": "assistant", "providerID": "openai", "modelID": "gpt-5.2-codex", + "tokens": {"input": 999, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}}) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ("mm_3", "ses_1", now_ms, now_ms, good + " trailing-garbage")) +# Completely broken row: not JSON at all. +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ("mm_4", "ses_1", now_ms, now_ms, "this is not json")) +conn.commit() +conn.close() +PY + +result=$(HOME="$MALFORMED_HOME" CODEX_HOME="$MALFORMED_HOME/.codex" XDG_CACHE_HOME="$MALFORMED_HOME/.cache" XDG_DATA_HOME="$MALFORMED_HOME/.local/share" \ + PATH="$MALFORMED_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "12" ]] || + fail "Codex collector counts good opencode rows past malformed ones" "$result" +pass "Codex collector counts good opencode rows past malformed ones" + +# An unwritable cache must not kill the collector: the record is the contract. +UNWRITABLE_HOME=$(mktemp -d) +trap 'rm -rf "$TEST_HOME" "$PI_HOME" "$OPENCODE_HOME" "$CACHE_HOME" "$FRESH_HOME" "$MALFORMED_HOME" "$UNWRITABLE_HOME"' EXIT +mkdir -p "$UNWRITABLE_HOME/bin" +cp "$TEST_HOME/bin/codex" "$UNWRITABLE_HOME/bin/codex" + +python3 - "$UNWRITABLE_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +db.parent.mkdir(parents=True, exist_ok=True) +conn = sqlite3.connect(db) +conn.execute("CREATE TABLE message (id text PRIMARY KEY, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL)") +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "u_1", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 3, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY + +# XDG_CACHE_HOME points at a regular file, so mkdir inside cache_root fails. +touch "$UNWRITABLE_HOME/not-a-dir" +result=$(HOME="$UNWRITABLE_HOME" CODEX_HOME="$UNWRITABLE_HOME/.codex" XDG_CACHE_HOME="$UNWRITABLE_HOME/not-a-dir" XDG_DATA_HOME="$UNWRITABLE_HOME/.local/share" \ + PATH="$UNWRITABLE_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "3" ]] || + fail "Codex collector still prints a complete record when the cache is unwritable" "$result" +pass "Codex collector still prints a complete record when the cache is unwritable" + +# A scan cut short by a database error (schema migration, transient lock, +# corruption) must not be cached as the whole story, or the missing usage +# would be suppressed for every reader until the cache expires. +INTERRUPTED_HOME=$(mktemp -d) +trap 'rm -rf "$TEST_HOME" "$PI_HOME" "$OPENCODE_HOME" "$CACHE_HOME" "$FRESH_HOME" "$MALFORMED_HOME" "$UNWRITABLE_HOME" "$INTERRUPTED_HOME"' EXIT +mkdir -p "$INTERRUPTED_HOME/bin" +cp "$TEST_HOME/bin/codex" "$INTERRUPTED_HOME/bin/codex" + +# A database without the message table makes the scan fail mid-flight. +python3 - "$INTERRUPTED_HOME/.local/share/opencode/opencode.db" <<'PY' +import sqlite3 +import sys +from pathlib import Path + +db = Path(sys.argv[1]) +db.parent.mkdir(parents=True, exist_ok=True) +conn = sqlite3.connect(db) +conn.execute("CREATE TABLE unrelated (id text PRIMARY KEY)") +conn.commit() +conn.close() +PY + +result=$(HOME="$INTERRUPTED_HOME" CODEX_HOME="$INTERRUPTED_HOME/.codex" XDG_CACHE_HOME="$INTERRUPTED_HOME/.cache" XDG_DATA_HOME="$INTERRUPTED_HOME/.local/share" \ + PATH="$INTERRUPTED_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex") + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "0" ]] || + fail "Codex collector reports what it could read from a broken database" "$result" +[[ -z $(ls "$INTERRUPTED_HOME/.cache/omarchy/agent-usage/"codex-scan-*.json 2>/dev/null) ]] || + fail "Codex collector must not cache an interrupted scan" "$result" + +# Once the database is whole again, the very next --limits-only run scans it +# instead of reusing a zero snapshot. +python3 - "$INTERRUPTED_HOME/.local/share/opencode/opencode.db" <<'PY' +import json +import sqlite3 +import sys +import time +from pathlib import Path + +db = Path(sys.argv[1]) +conn = sqlite3.connect(db) +conn.execute("CREATE TABLE message (id text PRIMARY KEY, session_id text NOT NULL, time_created integer NOT NULL, time_updated integer NOT NULL, data text NOT NULL)") +now_ms = int(time.time() * 1000) +conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", ( + "i_1", "ses_1", now_ms, now_ms, json.dumps({ + "role": "assistant", + "providerID": "openai", + "modelID": "gpt-5.2-codex", + "tokens": {"input": 9, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, + "time": {"created": now_ms}, + }), +)) +conn.commit() +conn.close() +PY + +result=$(HOME="$INTERRUPTED_HOME" CODEX_HOME="$INTERRUPTED_HOME/.codex" XDG_CACHE_HOME="$INTERRUPTED_HOME/.cache" XDG_DATA_HOME="$INTERRUPTED_HOME/.local/share" \ + PATH="$INTERRUPTED_HOME/bin:$PATH" "$ROOT/bin/omarchy-agent-usage-codex" --limits-only) + +[[ $(jq -r '.todayTotalTokens' <<<"$result") == "9" ]] || + fail "Codex collector does not reuse a snapshot from an interrupted scan" "$result" +pass "Codex collector does not cache an interrupted opencode scan"