From 5961a3ff5d19bf9f5931f79f54cc913b106e4649 Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Thu, 8 Oct 2026 11:59:48 -0400 Subject: [PATCH] Replace builder droplets stuck provisioning (#864) A droplet DigitalOcean still reports as "new" never registers a runner, but the controller counted it as one booting and created no replacement until MAX_AGE_MINUTES. A publish job sat queued for minutes behind one in mkc1. Delete droplets still provisioning after MAX_BOOT_MINUTES (10) and leave them out of the live count, so the same tick creates a replacement. --- ci/README.md | 2 +- ci/controller-box/controller.env.example | 2 ++ ci/controller.sh | 15 ++++++++++++--- tests/controller.sh | 3 +++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ci/README.md b/ci/README.md index fb31448..d0d0739 100644 --- a/ci/README.md +++ b/ci/README.md @@ -14,7 +14,7 @@ signing on merge exactly as before. - `controller.sh` — systemd timer every minute on a small always-on droplet. Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up to `MAX_DROPLETS`, deletes droplets that are powered off or older than - `MAX_AGE_MINUTES`. Builders go in any region DigitalOcean lists the size in + `MAX_AGE_MINUTES`, or still provisioning after `MAX_BOOT_MINUTES`. Builders go in any region DigitalOcean lists the size in stock in (`REGIONS` only sets which to try first); a refused create, logged with DigitalOcean's message, falls back to the next region, then the next of `SIZES`. No inbound endpoint. Plain curl against both APIs, no diff --git a/ci/controller-box/controller.env.example b/ci/controller-box/controller.env.example index 669e47d..c2b5979 100644 --- a/ci/controller-box/controller.env.example +++ b/ci/controller-box/controller.env.example @@ -11,6 +11,8 @@ SIZES="g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb" REGIONS= MAX_DROPLETS=6 MAX_AGE_MINUTES=200 +# Delete a droplet DigitalOcean still reports as provisioning after this long. +MAX_BOOT_MINUTES=10 LOCK=/run/omarchy-controller/lock # Operator public keys for root on every builder droplet (JSON array). # create.sh fills this from the operators' GitHub keys. diff --git a/ci/controller.sh b/ci/controller.sh index 60b15b1..d69f4e3 100755 --- a/ci/controller.sh +++ b/ci/controller.sh @@ -4,7 +4,8 @@ # Run from a systemd timer every minute on a small always-on droplet. No # inbound endpoint: it polls GitHub for queued jobs wanting our label, creates # one ephemeral droplet per job (up to MAX_DROPLETS), and deletes droplets -# that have powered off or exceeded MAX_AGE_MINUTES. The reaper does not +# that have powered off, exceeded MAX_AGE_MINUTES, or are still provisioning +# after MAX_BOOT_MINUTES. The reaper does not # trust its own bookkeeping: it lists by tag and acts on what DigitalOcean # reports. # @@ -30,6 +31,10 @@ REGIONS=${REGIONS:-${REGION:-}} IMAGE=${IMAGE:-ubuntu-24-04-x64} MAX_DROPLETS=${MAX_DROPLETS:-4} MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200} +# A droplet DigitalOcean still reports as "new" this long after creation is +# stuck provisioning. Left alone it counts as a runner booting, and holds a +# queued job until MAX_AGE_MINUTES. +MAX_BOOT_MINUTES=${MAX_BOOT_MINUTES:-10} RUNNER_VERSION=${RUNNER_VERSION:-2.337.0} CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml} # Operator public keys authorized on every builder (JSON array of strings). @@ -60,7 +65,8 @@ reap() { while read -r id status created; do [[ -n "$id" ]] || continue age=$(( (now - $(date -d "$created" +%s)) / 60 )) - if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )); then + if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )) || + { [[ $status == new ]] && (( age > MAX_BOOT_MINUTES )); }; then log "deleting droplet $id (status=$status age=${age}m)" do_api "droplets/$id" -X DELETE fi @@ -99,7 +105,10 @@ queued_jobs() { } live_droplets() { - do_api "droplets?tag_name=$TAG&per_page=200" | jq '[.droplets[] | select(.status != "off")] | length' + # Not the ones reap() just deleted: DigitalOcean can list them for a while. + do_api "droplets?tag_name=$TAG&per_page=200" | jq --argjson boot "$MAX_BOOT_MINUTES" ' + [.droplets[] | select(.status != "off") + | select(.status != "new" or (now - (.created_at | fromdateiso8601)) / 60 <= $boot)] | length' } busy_runners() { diff --git a/tests/controller.sh b/tests/controller.sh index c02c0db..cd21aa6 100755 --- a/tests/controller.sh +++ b/tests/controller.sh @@ -15,6 +15,7 @@ CONTROLLER_LIBRARY_ONLY=1 source "$ROOT/ci/controller.sh" CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT NOW=$(date -u +%FT%TZ) OLD=$(date -u -d '5 hours ago' +%FT%TZ) +STUCK=$(date -u -d '15 minutes ago' +%FT%TZ) # Scenario state: DROPLETS is "id status created" lines, QUEUED a count, # BUSY a count. @@ -54,6 +55,8 @@ DROPLETS="1 active $NOW" QUEUED=1 BUSY=1; run; check "one queued, one live but b DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0 DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1 DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1 +DROPLETS="1 new $STUCK" QUEUED=1 BUSY=0; run; check "droplet stuck provisioning reaped and replaced" 1 1 +DROPLETS="1 new $NOW" QUEUED=1 BUSY=0; run; check "droplet still provisioning is left to boot" 0 0 DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0 DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0 DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1