From 28050e8e75a207c0d04edd1a7891a9520d3919ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Fri, 17 Jul 2026 20:50:54 -0400 Subject: [PATCH] F9: smoke checklist + performance budgets in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docs/SMOKE.md: the F9 capability checklist — every carried-over harness capability mapped to at least one automated case (test target) or a manual probe with its observable, per the PRD acceptance rule. - scripts/bench.sh: enforces both PRD performance budgets and fails CI on a miss — `kigi --version` p95 <= 50ms via hyperfine, and TUI first frame <= 300ms measured through the pty harness (real vt100 emulator answering terminal queries) via the new scripts/first-frame.scenario.json. Local run: p95 10.1ms, first frame 134ms. - pty harness: StepOutcome gains elapsed_ms (stamped per step by the scripted runner) — a leading wait_for_text step's elapsed IS the spawn-to-first-paint latency the budget reads. - CI: new `perf` job (macOS + Linux) building the release binaries and running scripts/bench.sh. --- .github/workflows/ci.yml | 27 ++++++++ .../kigi-pager-pty-harness/src/scripted.rs | 11 +++- docs/SMOKE.md | 31 +++++++++ scripts/bench.sh | 64 +++++++++++++++++++ scripts/first-frame.scenario.json | 9 +++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 docs/SMOKE.md create mode 100755 scripts/bench.sh create mode 100644 scripts/first-frame.scenario.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9a57af..87e3a41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,3 +50,30 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: cargo test run: cargo test --workspace --locked + + perf: + name: performance budgets (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [macos-14, ubuntu-24.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install toolchain (rust-toolchain.toml) + run: rustup show + - name: Install dotslash (protoc launcher) + run: cargo install dotslash --locked + - uses: Swatinem/rust-cache@v2 + - name: Install hyperfine (macOS) + if: runner.os == 'macOS' + run: brew install hyperfine + - name: Install hyperfine (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y hyperfine + - name: Build release binaries + run: | + cargo build --release -p kigi-bin --locked + cargo build --release -p kigi-pager-pty-harness --bin pty-scenario --locked + - name: Enforce performance budgets + run: scripts/bench.sh target/release/kigi diff --git a/crates/codegen/kigi-pager-pty-harness/src/scripted.rs b/crates/codegen/kigi-pager-pty-harness/src/scripted.rs index 63048cd..954d520 100644 --- a/crates/codegen/kigi-pager-pty-harness/src/scripted.rs +++ b/crates/codegen/kigi-pager-pty-harness/src/scripted.rs @@ -585,6 +585,7 @@ impl ScriptedScenarioRunner { for (index, step) in scenario.steps.iter().enumerate() { let step_number = index + 1; + let step_started = std::time::Instant::now(); match run_step( &mut harness, &content, @@ -593,7 +594,8 @@ impl ScriptedScenarioRunner { step_number, step, ) { - Ok(outcome) => { + Ok(mut outcome) => { + outcome.elapsed_ms = step_started.elapsed().as_millis() as u64; report.artifacts.extend(outcome.artifacts.clone()); report.steps.push(outcome); } @@ -1189,6 +1191,11 @@ pub struct StepOutcome { pub action: String, pub status: StepStatus, pub message: Option, + /// Wall-clock time this step took. For a leading `WaitForText` step this + /// IS the pager's first-frame latency (spawn → text painted), which the + /// CI perf budget reads (scripts/bench.sh). + #[serde(default)] + pub elapsed_ms: u64, #[serde(default)] pub artifacts: Vec, } @@ -1200,6 +1207,7 @@ impl StepOutcome { action: action_name(action).to_owned(), status: StepStatus::Passed, message: None, + elapsed_ms: 0, artifacts: Vec::new(), } } @@ -1221,6 +1229,7 @@ impl StepOutcome { action: action_name(action).to_owned(), status: StepStatus::Failed, message: Some(message), + elapsed_ms: 0, artifacts: Vec::new(), } } diff --git a/docs/SMOKE.md b/docs/SMOKE.md new file mode 100644 index 0000000..1661600 --- /dev/null +++ b/docs/SMOKE.md @@ -0,0 +1,31 @@ +# Smoke checklist (PRD F9) + +F9 carries the grok-build harness capabilities forward unchanged. Each row +names the capability, the automated case that covers it (run with +`CARGO_INCREMENTAL=0`), and the manual probe when automation cannot reach it. +A release is smoke-clean when every row has at least one passing case. + +Legend: `auto` = covered by the named test target in CI; `manual` = run the +listed command and check the listed observable. + +| # | Capability | Case | How | +|---|------------|------|-----| +| 1 | Fullscreen TUI (mouse, themes, shortcuts, slash commands) | auto | `cargo test -p kigi-tui --lib` (welcome/menu/mouse/theme suites, 6.6k tests); manual spot: `kigi` → welcome moon renders, `ctrl+q` quits | +| 2 | Headless / script mode | auto | `cargo test -p kigi-tui --lib headless`; manual: `kigi -p "Reply OK"` prints the reply and exits 0 | +| 3 | ACP server | manual | `printf '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{"fs":{"readTextFile":false,"writeTextFile":false},"terminal":false}}}\n' \| kigi acp` → one JSON-RPC result line with `"protocolVersion":1` | +| 4 | MCP client (stdio/http/oauth) | auto | `cargo test -p kigi-mcp --lib`, `cargo test -p kigi-shell --lib util::config::mcp`; manual: `kigi mcp add t -- npx -y @modelcontextprotocol/server-everything` then `kigi mcp doctor t` | +| 5 | `--mcp-config-file` injection | auto | `cargo test -p kigi-shell --lib util::config::mcp::tests::cli_mcp`; manual: flag + `x.ai/mcp/servers_updated` notification lists the injected server | +| 6 | Skills | auto | `cargo test -p kigi-agent --lib skills` | +| 7 | Plugins (local load) | auto | `cargo test -p kigi-agent --lib plugins`; manual: `kigi plugin list` | +| 8 | Hooks | auto | `cargo test -p kigi-hooks --lib` and `cargo test -p kigi-shell --lib hooks` | +| 9 | Sandbox | auto | `cargo test -p kigi-sandbox --lib` | +| 10 | Checkpoint / session persistence | auto | `cargo test -p kigi-shell --lib session::` (persistence/rewind suites); manual: `kigi sessions` lists the last session, `kigi -c` resumes it | +| 11 | Worktrees | auto | `cargo test -p kigi-fast-worktree --lib`; manual: `kigi worktree list` | +| 12 | Mermaid rendering | auto | `cargo test -p kigi-mermaid --lib` | +| 13 | Crash handling | auto | `cargo test -p kigi-crash-handler --lib` | +| 14 | Kimi auth (device flow) | auto | `cargo test -p kigi-shell --lib auth::` (138 cases incl. live-shape fixtures); manual: `kigi login` completes in a browser, token lands in keyring service `kigi` | +| 15 | Inference (ChatCompletions Kimi dialect) | auto | `cargo test -p kigi-sampler` (incl. `test_kimi_wire`); e2e: `scratchpad` mock flow (write → run → answer), see AGENTS.md | +| 16 | Model catalog sync (`/models`) | auto | `cargo test -p kigi-shell --lib models_fetch` + `cargo test -p kigi-models --lib` | +| 17 | Search/fetch tools (OAuth-gated) | auto | `cargo test -p kigi-tools --lib web_search` and `--lib web_fetch` (Kimi wire contracts) | +| 18 | Performance budgets | auto | `scripts/bench.sh` — `kigi --version` p95 ≤ 50 ms, TUI first frame ≤ 300 ms (CI `perf` job) | +| 19 | Coexistence with official kimi-cli | manual | both logged in on one machine; kigi never reads/writes `~/.kimi` or keyring service `kimi-code` (F7 import is read-only; §9 requires a 24 h parallel-use pass) | diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100755 index 0000000..f87d6cf --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Performance budgets (PRD §7.3, enforced in CI from M2): +# 1. `kigi --version` p95 ≤ 50 ms (hyperfine) +# 2. TUI first frame ≤ 300 ms (PTY probe: spawn → first welcome paint) +# +# Usage: scripts/bench.sh [path-to-kigi-binary] +# Defaults to target/release/kigi. Exits non-zero on any budget miss. +set -euo pipefail + +KIGI_BIN="${1:-target/release/kigi}" +VERSION_P95_BUDGET_MS=50 +FIRST_FRAME_BUDGET_MS=300 + +if [[ ! -x "$KIGI_BIN" ]]; then + echo "error: kigi binary not found at $KIGI_BIN (build with: cargo build --release -p kigi-bin)" >&2 + exit 2 +fi + +echo "── budget 1: '$KIGI_BIN --version' p95 ≤ ${VERSION_P95_BUDGET_MS}ms ──" +json=$(mktemp) +hyperfine --warmup 3 --runs 30 --export-json "$json" "$KIGI_BIN --version" >/dev/null +p95_ms=$(python3 - "$json" <<'PY' +import json, sys, statistics +times = json.load(open(sys.argv[1]))["results"][0]["times"] +qs = statistics.quantiles(times, n=20) # qs[18] = p95 +print(f"{qs[18] * 1000:.1f}") +PY +) +rm -f "$json" +echo "p95 = ${p95_ms}ms" +awk -v p="$p95_ms" -v b="$VERSION_P95_BUDGET_MS" 'BEGIN { exit !(p <= b) }' || { + echo "FAIL: --version p95 ${p95_ms}ms exceeds ${VERSION_P95_BUDGET_MS}ms" >&2 + exit 1 +} + +echo "── budget 2: TUI first frame ≤ ${FIRST_FRAME_BUDGET_MS}ms ──" +# Measured through the pty harness (real vt100 emulator answering terminal +# queries): the leading wait_for_text step's elapsed_ms IS spawn → first +# painted welcome frame. See scripts/first-frame.scenario.json. +PTY_SCENARIO="${PTY_SCENARIO:-target/release/pty-scenario}" +if [[ ! -x "$PTY_SCENARIO" ]]; then + echo "error: pty-scenario not found at $PTY_SCENARIO (build with: cargo build --release -p kigi-pager-pty-harness --bin pty-scenario)" >&2 + exit 2 +fi +artifacts=$(mktemp -d) +report=$("$PTY_SCENARIO" \ + --scenario scripts/first-frame.scenario.json \ + --binary "$(cd "$(dirname "$KIGI_BIN")" && pwd)/$(basename "$KIGI_BIN")" \ + --artifacts "$artifacts") +rm -rf "$artifacts" +first_ms=$(python3 - "$report" <<'PY' +import json, sys +d = json.loads(sys.argv[1][sys.argv[1].index("{"):]) +assert d["status"] == "passed", f"scenario failed: {d}" +print(d["steps"][0]["elapsed_ms"]) +PY +) +echo "first frame = ${first_ms}ms" +awk -v p="$first_ms" -v b="$FIRST_FRAME_BUDGET_MS" 'BEGIN { exit !(p <= b) }' || { + echo "FAIL: first frame ${first_ms}ms exceeds ${FIRST_FRAME_BUDGET_MS}ms" >&2 + exit 1 +} + +echo "PASS: all performance budgets met" diff --git a/scripts/first-frame.scenario.json b/scripts/first-frame.scenario.json new file mode 100644 index 0000000..6871554 --- /dev/null +++ b/scripts/first-frame.scenario.json @@ -0,0 +1,9 @@ +{ + "name": "first-frame", + "description": "CI perf budget: time from pager spawn to the first painted welcome frame (PRD: <= 300ms).", + "terminal": { "rows": 40, "cols": 120, "respond_to_queries": true }, + "environment": { "env": [ { "key": "KIGI_DISABLE_KEYRING", "value": "1" } ] }, + "steps": [ + { "action": "wait_for_text", "text": "Kigi", "timeout_ms": 10000 } + ] +}