F9: smoke checklist + performance budgets in CI
- docs/SMOKE.md: the F9 capability checklist — every carried-over harness capability mapped to at least one automated case (test target) or a manual probe with its observable, per the PRD acceptance rule. - scripts/bench.sh: enforces both PRD performance budgets and fails CI on a miss — `kigi --version` p95 <= 50ms via hyperfine, and TUI first frame <= 300ms measured through the pty harness (real vt100 emulator answering terminal queries) via the new scripts/first-frame.scenario.json. Local run: p95 10.1ms, first frame 134ms. - pty harness: StepOutcome gains elapsed_ms (stamped per step by the scripted runner) — a leading wait_for_text step's elapsed IS the spawn-to-first-paint latency the budget reads. - CI: new `perf` job (macOS + Linux) building the release binaries and running scripts/bench.sh.
This commit is contained in:
@@ -50,3 +50,30 @@ jobs:
|
|||||||
- uses: Swatinem/rust-cache@v2
|
- uses: Swatinem/rust-cache@v2
|
||||||
- name: cargo test
|
- name: cargo test
|
||||||
run: cargo test --workspace --locked
|
run: cargo test --workspace --locked
|
||||||
|
|
||||||
|
perf:
|
||||||
|
name: performance budgets (${{ matrix.os }})
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os: [macos-14, ubuntu-24.04]
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Install toolchain (rust-toolchain.toml)
|
||||||
|
run: rustup show
|
||||||
|
- name: Install dotslash (protoc launcher)
|
||||||
|
run: cargo install dotslash --locked
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- name: Install hyperfine (macOS)
|
||||||
|
if: runner.os == 'macOS'
|
||||||
|
run: brew install hyperfine
|
||||||
|
- name: Install hyperfine (Linux)
|
||||||
|
if: runner.os == 'Linux'
|
||||||
|
run: sudo apt-get update && sudo apt-get install -y hyperfine
|
||||||
|
- name: Build release binaries
|
||||||
|
run: |
|
||||||
|
cargo build --release -p kigi-bin --locked
|
||||||
|
cargo build --release -p kigi-pager-pty-harness --bin pty-scenario --locked
|
||||||
|
- name: Enforce performance budgets
|
||||||
|
run: scripts/bench.sh target/release/kigi
|
||||||
|
|||||||
@@ -585,6 +585,7 @@ impl ScriptedScenarioRunner {
|
|||||||
|
|
||||||
for (index, step) in scenario.steps.iter().enumerate() {
|
for (index, step) in scenario.steps.iter().enumerate() {
|
||||||
let step_number = index + 1;
|
let step_number = index + 1;
|
||||||
|
let step_started = std::time::Instant::now();
|
||||||
match run_step(
|
match run_step(
|
||||||
&mut harness,
|
&mut harness,
|
||||||
&content,
|
&content,
|
||||||
@@ -593,7 +594,8 @@ impl ScriptedScenarioRunner {
|
|||||||
step_number,
|
step_number,
|
||||||
step,
|
step,
|
||||||
) {
|
) {
|
||||||
Ok(outcome) => {
|
Ok(mut outcome) => {
|
||||||
|
outcome.elapsed_ms = step_started.elapsed().as_millis() as u64;
|
||||||
report.artifacts.extend(outcome.artifacts.clone());
|
report.artifacts.extend(outcome.artifacts.clone());
|
||||||
report.steps.push(outcome);
|
report.steps.push(outcome);
|
||||||
}
|
}
|
||||||
@@ -1189,6 +1191,11 @@ pub struct StepOutcome {
|
|||||||
pub action: String,
|
pub action: String,
|
||||||
pub status: StepStatus,
|
pub status: StepStatus,
|
||||||
pub message: Option<String>,
|
pub message: Option<String>,
|
||||||
|
/// Wall-clock time this step took. For a leading `WaitForText` step this
|
||||||
|
/// IS the pager's first-frame latency (spawn → text painted), which the
|
||||||
|
/// CI perf budget reads (scripts/bench.sh).
|
||||||
|
#[serde(default)]
|
||||||
|
pub elapsed_ms: u64,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub artifacts: Vec<VisualArtifact>,
|
pub artifacts: Vec<VisualArtifact>,
|
||||||
}
|
}
|
||||||
@@ -1200,6 +1207,7 @@ impl StepOutcome {
|
|||||||
action: action_name(action).to_owned(),
|
action: action_name(action).to_owned(),
|
||||||
status: StepStatus::Passed,
|
status: StepStatus::Passed,
|
||||||
message: None,
|
message: None,
|
||||||
|
elapsed_ms: 0,
|
||||||
artifacts: Vec::new(),
|
artifacts: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1221,6 +1229,7 @@ impl StepOutcome {
|
|||||||
action: action_name(action).to_owned(),
|
action: action_name(action).to_owned(),
|
||||||
status: StepStatus::Failed,
|
status: StepStatus::Failed,
|
||||||
message: Some(message),
|
message: Some(message),
|
||||||
|
elapsed_ms: 0,
|
||||||
artifacts: Vec::new(),
|
artifacts: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# Smoke checklist (PRD F9)
|
||||||
|
|
||||||
|
F9 carries the grok-build harness capabilities forward unchanged. Each row
|
||||||
|
names the capability, the automated case that covers it (run with
|
||||||
|
`CARGO_INCREMENTAL=0`), and the manual probe when automation cannot reach it.
|
||||||
|
A release is smoke-clean when every row has at least one passing case.
|
||||||
|
|
||||||
|
Legend: `auto` = covered by the named test target in CI; `manual` = run the
|
||||||
|
listed command and check the listed observable.
|
||||||
|
|
||||||
|
| # | Capability | Case | How |
|
||||||
|
|---|------------|------|-----|
|
||||||
|
| 1 | Fullscreen TUI (mouse, themes, shortcuts, slash commands) | auto | `cargo test -p kigi-tui --lib` (welcome/menu/mouse/theme suites, 6.6k tests); manual spot: `kigi` → welcome moon renders, `ctrl+q` quits |
|
||||||
|
| 2 | Headless / script mode | auto | `cargo test -p kigi-tui --lib headless`; manual: `kigi -p "Reply OK"` prints the reply and exits 0 |
|
||||||
|
| 3 | ACP server | manual | `printf '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{"fs":{"readTextFile":false,"writeTextFile":false},"terminal":false}}}\n' \| kigi acp` → one JSON-RPC result line with `"protocolVersion":1` |
|
||||||
|
| 4 | MCP client (stdio/http/oauth) | auto | `cargo test -p kigi-mcp --lib`, `cargo test -p kigi-shell --lib util::config::mcp`; manual: `kigi mcp add t -- npx -y @modelcontextprotocol/server-everything` then `kigi mcp doctor t` |
|
||||||
|
| 5 | `--mcp-config-file` injection | auto | `cargo test -p kigi-shell --lib util::config::mcp::tests::cli_mcp`; manual: flag + `x.ai/mcp/servers_updated` notification lists the injected server |
|
||||||
|
| 6 | Skills | auto | `cargo test -p kigi-agent --lib skills` |
|
||||||
|
| 7 | Plugins (local load) | auto | `cargo test -p kigi-agent --lib plugins`; manual: `kigi plugin list` |
|
||||||
|
| 8 | Hooks | auto | `cargo test -p kigi-hooks --lib` and `cargo test -p kigi-shell --lib hooks` |
|
||||||
|
| 9 | Sandbox | auto | `cargo test -p kigi-sandbox --lib` |
|
||||||
|
| 10 | Checkpoint / session persistence | auto | `cargo test -p kigi-shell --lib session::` (persistence/rewind suites); manual: `kigi sessions` lists the last session, `kigi -c` resumes it |
|
||||||
|
| 11 | Worktrees | auto | `cargo test -p kigi-fast-worktree --lib`; manual: `kigi worktree list` |
|
||||||
|
| 12 | Mermaid rendering | auto | `cargo test -p kigi-mermaid --lib` |
|
||||||
|
| 13 | Crash handling | auto | `cargo test -p kigi-crash-handler --lib` |
|
||||||
|
| 14 | Kimi auth (device flow) | auto | `cargo test -p kigi-shell --lib auth::` (138 cases incl. live-shape fixtures); manual: `kigi login` completes in a browser, token lands in keyring service `kigi` |
|
||||||
|
| 15 | Inference (ChatCompletions Kimi dialect) | auto | `cargo test -p kigi-sampler` (incl. `test_kimi_wire`); e2e: `scratchpad` mock flow (write → run → answer), see AGENTS.md |
|
||||||
|
| 16 | Model catalog sync (`/models`) | auto | `cargo test -p kigi-shell --lib models_fetch` + `cargo test -p kigi-models --lib` |
|
||||||
|
| 17 | Search/fetch tools (OAuth-gated) | auto | `cargo test -p kigi-tools --lib web_search` and `--lib web_fetch` (Kimi wire contracts) |
|
||||||
|
| 18 | Performance budgets | auto | `scripts/bench.sh` — `kigi --version` p95 ≤ 50 ms, TUI first frame ≤ 300 ms (CI `perf` job) |
|
||||||
|
| 19 | Coexistence with official kimi-cli | manual | both logged in on one machine; kigi never reads/writes `~/.kimi` or keyring service `kimi-code` (F7 import is read-only; §9 requires a 24 h parallel-use pass) |
|
||||||
Executable
+64
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Performance budgets (PRD §7.3, enforced in CI from M2):
|
||||||
|
# 1. `kigi --version` p95 ≤ 50 ms (hyperfine)
|
||||||
|
# 2. TUI first frame ≤ 300 ms (PTY probe: spawn → first welcome paint)
|
||||||
|
#
|
||||||
|
# Usage: scripts/bench.sh [path-to-kigi-binary]
|
||||||
|
# Defaults to target/release/kigi. Exits non-zero on any budget miss.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
KIGI_BIN="${1:-target/release/kigi}"
|
||||||
|
VERSION_P95_BUDGET_MS=50
|
||||||
|
FIRST_FRAME_BUDGET_MS=300
|
||||||
|
|
||||||
|
if [[ ! -x "$KIGI_BIN" ]]; then
|
||||||
|
echo "error: kigi binary not found at $KIGI_BIN (build with: cargo build --release -p kigi-bin)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "── budget 1: '$KIGI_BIN --version' p95 ≤ ${VERSION_P95_BUDGET_MS}ms ──"
|
||||||
|
json=$(mktemp)
|
||||||
|
hyperfine --warmup 3 --runs 30 --export-json "$json" "$KIGI_BIN --version" >/dev/null
|
||||||
|
p95_ms=$(python3 - "$json" <<'PY'
|
||||||
|
import json, sys, statistics
|
||||||
|
times = json.load(open(sys.argv[1]))["results"][0]["times"]
|
||||||
|
qs = statistics.quantiles(times, n=20) # qs[18] = p95
|
||||||
|
print(f"{qs[18] * 1000:.1f}")
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
rm -f "$json"
|
||||||
|
echo "p95 = ${p95_ms}ms"
|
||||||
|
awk -v p="$p95_ms" -v b="$VERSION_P95_BUDGET_MS" 'BEGIN { exit !(p <= b) }' || {
|
||||||
|
echo "FAIL: --version p95 ${p95_ms}ms exceeds ${VERSION_P95_BUDGET_MS}ms" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "── budget 2: TUI first frame ≤ ${FIRST_FRAME_BUDGET_MS}ms ──"
|
||||||
|
# Measured through the pty harness (real vt100 emulator answering terminal
|
||||||
|
# queries): the leading wait_for_text step's elapsed_ms IS spawn → first
|
||||||
|
# painted welcome frame. See scripts/first-frame.scenario.json.
|
||||||
|
PTY_SCENARIO="${PTY_SCENARIO:-target/release/pty-scenario}"
|
||||||
|
if [[ ! -x "$PTY_SCENARIO" ]]; then
|
||||||
|
echo "error: pty-scenario not found at $PTY_SCENARIO (build with: cargo build --release -p kigi-pager-pty-harness --bin pty-scenario)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
artifacts=$(mktemp -d)
|
||||||
|
report=$("$PTY_SCENARIO" \
|
||||||
|
--scenario scripts/first-frame.scenario.json \
|
||||||
|
--binary "$(cd "$(dirname "$KIGI_BIN")" && pwd)/$(basename "$KIGI_BIN")" \
|
||||||
|
--artifacts "$artifacts")
|
||||||
|
rm -rf "$artifacts"
|
||||||
|
first_ms=$(python3 - "$report" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
d = json.loads(sys.argv[1][sys.argv[1].index("{"):])
|
||||||
|
assert d["status"] == "passed", f"scenario failed: {d}"
|
||||||
|
print(d["steps"][0]["elapsed_ms"])
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
echo "first frame = ${first_ms}ms"
|
||||||
|
awk -v p="$first_ms" -v b="$FIRST_FRAME_BUDGET_MS" 'BEGIN { exit !(p <= b) }' || {
|
||||||
|
echo "FAIL: first frame ${first_ms}ms exceeds ${FIRST_FRAME_BUDGET_MS}ms" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "PASS: all performance budgets met"
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"name": "first-frame",
|
||||||
|
"description": "CI perf budget: time from pager spawn to the first painted welcome frame (PRD: <= 300ms).",
|
||||||
|
"terminal": { "rows": 40, "cols": 120, "respond_to_queries": true },
|
||||||
|
"environment": { "env": [ { "key": "KIGI_DISABLE_KEYRING", "value": "1" } ] },
|
||||||
|
"steps": [
|
||||||
|
{ "action": "wait_for_text", "text": "Kigi", "timeout_ms": 10000 }
|
||||||
|
]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user