Ryan Hughes
7e86af28b3
The controller created every builder without a DigitalOcean account key, so DigitalOcean set a root password, emailed it to the token's owner, and turned password logins on: one email and one password-reachable box per builder, 146 of them on Oct 8. DO_SSH_KEYS lists account key IDs attached at creation (Ryan, DHH and Emir in the template). It needs ssh_key:read on the controller's token; without it every create is refused with 403, so it stays empty until the token has it. Builders also turn password SSH off in cloud-init, which holds even while no account key is attached, and Emir joins the operators whose GitHub keys reach root.
141 lines
7.8 KiB
Bash
Executable File
141 lines
7.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Self-test for ci/controller.sh: every decision, no cloud.
|
|
#
|
|
# The controller's two API functions are overridden with canned responses and
|
|
# a recorder, then each scenario asserts which creates and deletes it issued.
|
|
set -euo pipefail
|
|
ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
|
|
|
export REPO=o/r DIGITALOCEAN_TOKEN=x GITHUB_TOKEN=x
|
|
export CLOUD_INIT="$ROOT/ci/runner-cloud-init.yaml" LOCK=/tmp/controller-test.lock
|
|
CONTROLLER_LIBRARY_ONLY=1 source "$ROOT/ci/controller.sh"
|
|
|
|
# Calls are recorded to a file: the controller invokes the API functions
|
|
# inside command substitutions, and a subshell cannot append to an array.
|
|
CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT
|
|
NOW=$(date -u +%FT%TZ)
|
|
OLD=$(date -u -d '5 hours ago' +%FT%TZ)
|
|
STUCK=$(date -u -d '15 minutes ago' +%FT%TZ)
|
|
|
|
# Scenario state: DROPLETS is "id status created" lines, QUEUED a count,
|
|
# BUSY a count.
|
|
do_api() {
|
|
local path=$1; shift
|
|
echo "do $path $*" >>"$CALLS_FILE"
|
|
case "$path" in
|
|
sizes\?*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;;
|
|
droplets\?*) printf '%s\n' "$DROPLETS" | jq -Rs '{droplets: [split("\n")[] | select(length>0) | split(" ") | {id: .[0]|tonumber, status: .[1], created_at: .[2]}]}' ;;
|
|
droplets) echo '{"droplet":{"id":999}}' ;;
|
|
droplets/*) echo '{}' ;;
|
|
esac
|
|
}
|
|
gh_api() {
|
|
local path=$1; shift
|
|
echo "gh $path $*" >>"$CALLS_FILE"
|
|
case "$path" in
|
|
*/actions/runs\?status=in_progress*) echo '{"workflow_runs":[]}' ;;
|
|
*/actions/runs\?*) jq -nc --argjson n "$QUEUED" '{workflow_runs: [range($n) | {id: .}]}' ;;
|
|
*/actions/runs/*/jobs\?*) echo '{"jobs":[{"id":'"$(echo "$path" | cut -d/ -f6)"',"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;;
|
|
*/actions/runners\?*) jq -nc --argjson n "$BUSY" '{runners: [range($n) | {busy: true, labels: [{name: "omarchy-builder"}]}]}' ;;
|
|
*/registration-token) echo '{"token":"T"}' ;;
|
|
esac
|
|
}
|
|
|
|
creates() { grep -c '^do droplets -X POST' "$CALLS_FILE" || true; }
|
|
deletes() { grep -c '^do droplets/.* -X DELETE' "$CALLS_FILE" || true; }
|
|
run() { : >"$CALLS_FILE"; controller_tick >/dev/null; }
|
|
check() { # check <name> <expected creates> <expected deletes>
|
|
local c d; c=$(creates); d=$(deletes)
|
|
if [[ "$c" == "$2" && "$d" == "$3" ]]; then echo "PASS: $1"; else echo "FAIL: $1 (creates=$c want $2, deletes=$d want $3)"; cat "$CALLS_FILE"; exit 1; fi
|
|
}
|
|
|
|
DROPLETS="" QUEUED=0 BUSY=0; run; check "idle: nothing queued, nothing to reap" 0 0
|
|
DROPLETS="" QUEUED=2 BUSY=0; run; check "two queued, none live: create two" 2 0
|
|
DROPLETS="1 active $NOW" QUEUED=1 BUSY=1; run; check "one queued, one live but busy: create one" 1 0
|
|
DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0
|
|
DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1
|
|
DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1
|
|
DROPLETS="1 new $STUCK" QUEUED=1 BUSY=0; run; check "droplet stuck provisioning reaped and replaced" 1 1
|
|
DROPLETS="1 new $NOW" QUEUED=1 BUSY=0; run; check "droplet still provisioning is left to boot" 0 0
|
|
DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0
|
|
DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0
|
|
DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1
|
|
|
|
# A large in-progress matrix has no waiting jobs on page one. The queued
|
|
# workflow is on page two of the run listing; the same run can appear in
|
|
# both status queries while GitHub updates it, so count its jobs once.
|
|
gh_api() {
|
|
case "$1" in
|
|
*runs?status=queued*page=1) jq -nc '{workflow_runs: [range(100) | {id: .}]}' ;;
|
|
*runs?status=queued*page=2) echo '{"workflow_runs":[{"id":999}]}' ;;
|
|
*runs?status=in_progress*) echo '{"workflow_runs":[{"id":999},{"id":1000}]}' ;;
|
|
*runs/999/jobs*page=1) jq -nc '{jobs: [range(100) | {id: .,status:"completed",labels:["omarchy-builder"]}]}' ;;
|
|
*runs/999/jobs*page=2) echo '{"jobs":[{"id":9991,"status":"queued","labels":["omarchy-builder"]}]}' ;;
|
|
*runs/1000/jobs*) echo '{"jobs":[{"id":10001,"status":"queued","labels":["omarchy-builder"]},{"id":10002,"status":"queued","labels":["ubuntu-latest"]},{"id":10003,"status":"in_progress","labels":["omarchy-builder"]}]}' ;;
|
|
*jobs*) echo '{"jobs":[]}' ;;
|
|
*) echo "Unexpected API request: $1" >&2; return 1 ;;
|
|
esac
|
|
}
|
|
[[ $(queued_jobs) == 2 ]] || { echo "FAIL: full queue across pages and workflow states"; exit 1; }
|
|
echo "PASS: later run/job pages and in-progress workflows count each waiting builder once"
|
|
|
|
gh_api() {
|
|
case "$1" in
|
|
*runs?status=queued*) echo '{"workflow_runs":[{"id":1}]}' ;;
|
|
*runs?status=in_progress*) echo '{"workflow_runs":[]}' ;;
|
|
*jobs*page=1) jq -nc '{jobs:[range(100)|{id:.,status:"completed",labels:[]}]}' ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
if queued_jobs >/dev/null; then echo "FAIL: a failed later page looks like an empty queue"; exit 1; fi
|
|
echo "PASS: API failures stop the queue query"
|
|
|
|
# The create body must carry the tag (reaper scope) and substituted user-data.
|
|
BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT
|
|
do_api() {
|
|
case "$1" in
|
|
droplets) printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}' ;;
|
|
sizes*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;;
|
|
*) echo '{"droplets":[]}' ;;
|
|
esac
|
|
}
|
|
gh_api() { echo '{"token":"TOK"}'; }
|
|
CANDIDATES=""; create_droplet >/dev/null
|
|
jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.user_data | test("--token \"TOK\"")) and (.user_data | test("__") | not)' "$BODY_FILE" >/dev/null \
|
|
&& echo "PASS: create body carries tag, size, substituted user-data" \
|
|
|| { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; }
|
|
|
|
# Account keys reach the create, so DigitalOcean emails no root password.
|
|
CANDIDATES=""; DO_SSH_KEYS='[123]' create_droplet >/dev/null
|
|
jq -e '.ssh_keys == [123]' "$BODY_FILE" >/dev/null \
|
|
&& echo "PASS: create body attaches DO_SSH_KEYS" \
|
|
|| { echo "FAIL: create body ssh_keys"; jq .ssh_keys "$BODY_FILE"; exit 1; }
|
|
|
|
# Sizes are tried in SIZES order, each in the regions DigitalOcean lists it
|
|
# in stock (REGIONS first); a 422 falls through to the next pair with the
|
|
# refusal's message logged, and a refused pair is not retried in the tick.
|
|
# When every pair is refused, the create fails.
|
|
do_api() {
|
|
case "$1" in
|
|
sizes*) echo '{"sizes":[
|
|
{"slug":"small","available":true,"regions":["r1","r2"]},
|
|
{"slug":"gone","available":false,"regions":["r1"]},
|
|
{"slug":"big","available":true,"regions":["r3"]}]}' ;;
|
|
droplets)
|
|
local pair; pair=$(jq -r '"\(.size)@\(.region)"' <<< "${*: -1}"); echo "$pair" >>"$CALLS_FILE"
|
|
[[ $pair == big@r3 ]] && { echo '{"droplet":{"id":2}}'; return 0; }
|
|
echo '{"id":"unprocessable_entity","message":"Size is not available in this region."}'; return 22 ;;
|
|
esac
|
|
}
|
|
: >"$CALLS_FILE"; CANDIDATES=""
|
|
out=$(SIZES="small gone big" REGIONS="r2"; create_droplet; create_droplet)
|
|
[[ $(paste -sd' ' "$CALLS_FILE") == "small@r2 small@r1 big@r3 big@r3" \
|
|
&& $out == *"small in r2 refused: Size is not available in this region."* && $out == *"created droplet 2"* ]] \
|
|
&& echo "PASS: a refused size falls back to the next region, then the next size, logging why" \
|
|
|| { echo "FAIL: size and region fallback"; echo "$out"; cat "$CALLS_FILE"; exit 1; }
|
|
CANDIDATES=""
|
|
if out=$(SIZES="small gone" create_droplet); then echo "FAIL: every pair refused looks like a create"; exit 1; fi
|
|
[[ $out == *"no size in 'small gone' can be created in any region"* ]] \
|
|
&& echo "PASS: every size refused everywhere fails the create" \
|
|
|| { echo "FAIL: all-refused message"; echo "$out"; exit 1; }
|