Replace builder droplets stuck provisioning (#864)

A droplet DigitalOcean still reports as "new" never registers a runner, but
the controller counted it as one booting and created no replacement until
MAX_AGE_MINUTES. A publish job sat queued for minutes behind one in mkc1.

Delete droplets still provisioning after MAX_BOOT_MINUTES (10) and leave them
out of the live count, so the same tick creates a replacement.

ci/README.md

@@ -14,7 +14,7 @@ signing on merge exactly as before.
 - `controller.sh` — systemd timer every minute on a small always-on droplet.
   Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up
   to `MAX_DROPLETS`, deletes droplets that are powered off or older than
-  `MAX_AGE_MINUTES`. Builders go in any region DigitalOcean lists the size in
+  `MAX_AGE_MINUTES`, or still provisioning after `MAX_BOOT_MINUTES`. Builders go in any region DigitalOcean lists the size in
   stock in (`REGIONS` only sets which to try first); a refused create, logged
   with DigitalOcean's message, falls back to the next region, then the next
   of `SIZES`. No inbound endpoint. Plain curl against both APIs, no

ci/controller-box/controller.env.example

@@ -11,6 +11,8 @@ SIZES="g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb"
 REGIONS=
 MAX_DROPLETS=6
 MAX_AGE_MINUTES=200
+# Delete a droplet DigitalOcean still reports as provisioning after this long.
+MAX_BOOT_MINUTES=10
 LOCK=/run/omarchy-controller/lock
 # Operator public keys for root on every builder droplet (JSON array).
 # create.sh fills this from the operators' GitHub keys.

ci/controller.sh

@@ -4,7 +4,8 @@
 # Run from a systemd timer every minute on a small always-on droplet. No
 # inbound endpoint: it polls GitHub for queued jobs wanting our label, creates
 # one ephemeral droplet per job (up to MAX_DROPLETS), and deletes droplets
-# that have powered off or exceeded MAX_AGE_MINUTES. The reaper does not
+# that have powered off, exceeded MAX_AGE_MINUTES, or are still provisioning
+# after MAX_BOOT_MINUTES. The reaper does not
 # trust its own bookkeeping: it lists by tag and acts on what DigitalOcean
 # reports.
 #
@@ -30,6 +31,10 @@ REGIONS=${REGIONS:-${REGION:-}}
 IMAGE=${IMAGE:-ubuntu-24-04-x64}
 MAX_DROPLETS=${MAX_DROPLETS:-4}
 MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200}
+# A droplet DigitalOcean still reports as "new" this long after creation is
+# stuck provisioning. Left alone it counts as a runner booting, and holds a
+# queued job until MAX_AGE_MINUTES.
+MAX_BOOT_MINUTES=${MAX_BOOT_MINUTES:-10}
 RUNNER_VERSION=${RUNNER_VERSION:-2.337.0}
 CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml}
 # Operator public keys authorized on every builder (JSON array of strings).
@@ -60,7 +65,8 @@ reap() {
   while read -r id status created; do
     [[ -n "$id" ]] || continue
     age=$(( (now - $(date -d "$created" +%s)) / 60 ))
-    if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )); then
+    if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )) ||
+       { [[ $status == new ]] && (( age > MAX_BOOT_MINUTES )); }; then
       log "deleting droplet $id (status=$status age=${age}m)"
       do_api "droplets/$id" -X DELETE
     fi
@@ -99,7 +105,10 @@ queued_jobs() {
 }
 
 live_droplets() {
-  do_api "droplets?tag_name=$TAG&per_page=200" | jq '[.droplets[] | select(.status != "off")] | length'
+  # Not the ones reap() just deleted: DigitalOcean can list them for a while.
+  do_api "droplets?tag_name=$TAG&per_page=200" | jq --argjson boot "$MAX_BOOT_MINUTES" '
+    [.droplets[] | select(.status != "off")
+      | select(.status != "new" or (now - (.created_at | fromdateiso8601)) / 60 <= $boot)] | length'
 }
 
 busy_runners() {

tests/controller.sh

@@ -15,6 +15,7 @@ CONTROLLER_LIBRARY_ONLY=1 source "$ROOT/ci/controller.sh"
 CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT
 NOW=$(date -u +%FT%TZ)
 OLD=$(date -u -d '5 hours ago' +%FT%TZ)
+STUCK=$(date -u -d '15 minutes ago' +%FT%TZ)
 
 # Scenario state: DROPLETS is "id status created" lines, QUEUED a count,
 # BUSY a count.
@@ -54,6 +55,8 @@ DROPLETS="1 active $NOW" QUEUED=1 BUSY=1; run; check "one queued, one live but b
 DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0
 DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1
 DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1
+DROPLETS="1 new $STUCK" QUEUED=1 BUSY=0; run; check "droplet stuck provisioning reaped and replaced" 1 1
+DROPLETS="1 new $NOW" QUEUED=1 BUSY=0; run; check "droplet still provisioning is left to boot" 0 0
 DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0
 DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0
 DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1