Keep builders coming when DigitalOcean sells out a size or region (#861)

* Keep builders coming when DigitalOcean sells out a droplet size ric1 sold out of g5-32vcpu-64gb-50gb, and every create came back 422. curl -f dropped the reason and set -e ended the tick, so builders only appeared when capacity happened to free up, and the journal showed nothing but "curl: (22)". Try each of SIZES in turn, logging DigitalOcean's refusal message, and fail the tick only when every size is refused. Builders now power off however start.sh exits, so a failed registration is reaped instead of counting as a booting runner until MAX_AGE_MINUTES. The controller unit pulls the checkout before each tick, so merged controller fixes reach the box. * Create builders in any region that has the size in stock ric1 sold out of g5-32vcpu-128gb-50gb within minutes of the box switching to it. Builders need nothing from a particular region, so read DigitalOcean's size catalog once per tick and try each of SIZES in every region it lists in stock, REGIONS first if set. A refused pair is dropped for the rest of the tick.
2026-10-08 10:50:06 -04:00
1 parent 024943141d
commit 128c5647ba
@@ -12,9 +12,12 @@ signing on merge exactly as before.
 - `runner-cloud-init.yaml` — Ubuntu 24.04 user-data: docker + buildx, the
  GitHub runner registered `--ephemeral`, runs one job, powers off.
 - `controller.sh` — systemd timer every minute on a small always-on droplet.
-  Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet (ric1) per job up
+  Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up
  to `MAX_DROPLETS`, deletes droplets that are powered off or older than
-  `MAX_AGE_MINUTES`. No inbound endpoint. Plain curl against both APIs, no
+  `MAX_AGE_MINUTES`. Builders go in any region DigitalOcean lists the size in
+  stock in (`REGIONS` only sets which to try first); a refused create, logged
+  with DigitalOcean's message, falls back to the next region, then the next
+  of `SIZES`. No inbound endpoint. Plain curl against both APIs, no
  doctl and no gh: a token in the environment cannot pick the wrong account
  the way a saved doctl context can. Needs curl and jq.
  `tests/controller.sh` exercises every decision against canned responses.
@@ -31,6 +34,13 @@ Administration read+write (registration tokens). The DO token is baked into
 the box's env file, so it is the account that pays for builder droplets.
 Watch it with `journalctl -u omarchy-controller -f` on the box.

+Each tick pulls the box's checkout first, so a merged `controller.sh` is live
+within a minute. The unit and timer are copies made at creation; after
+changing them, on the box:
+
+    cp /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.{service,timer} /etc/systemd/system/
+    systemctl daemon-reload
+
 ## What the spike proved (2026-09-17, fork ryanrhughes/omarchy-pkgs)

 - `bin/build` works from a bare clone: with no local published tree it
@@ -5,8 +5,10 @@ GITHUB_TOKEN=github_pat_...
 REPO=omacom/omarchy-pkgs
 LABEL=omarchy-builder
 TAG=omarchy-builder
-REGION=ric1
-SIZE=g5-32vcpu-64gb-50gb
+# Builder sizes, tried in order in any region that has them in stock.
+SIZES="g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb"
+# Optional: regions to try first, e.g. "ric1". Empty means any.
+REGIONS=
 MAX_DROPLETS=6
 MAX_AGE_MINUTES=200
 LOCK=/run/omarchy-controller/lock
@@ -7,6 +7,9 @@ Wants=network-online.target
 Type=oneshot
 User=controller
 EnvironmentFile=/etc/omarchy-controller.env
+# Run the branch's current controller, not the one cloned when the box was
+# built. As root (the checkout's owner); a failed pull keeps the last one.
+ExecStartPre=-+/usr/bin/git -C /opt/omarchy-pkgs pull --ff-only --quiet
 ExecStart=/opt/omarchy-pkgs/ci/controller.sh
 # The reaper's safety net is time, not state; a hung tick must not hold the lock.
 TimeoutStartSec=240
@@ -21,8 +21,12 @@ REPO=${REPO:?owner/name}
 : "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}"
 LABEL=${LABEL:-omarchy-builder}
 TAG=${TAG:-omarchy-builder}
-REGION=${REGION:-ric1}
-SIZE=${SIZE:-g5-32vcpu-64gb-50gb}
+# Sizes to try, in order, in any region DigitalOcean lists them in stock. A
+# size can sell out in a region for hours; the create is then refused with
+# 422 and the next region, then the next size, is tried. REGIONS only orders
+# the regions tried first. SIZE and REGION, if set, are one-item lists.
+SIZES=${SIZES:-${SIZE:-g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb}}
+REGIONS=${REGIONS:-${REGION:-}}
 IMAGE=${IMAGE:-ubuntu-24-04-x64}
 MAX_DROPLETS=${MAX_DROPLETS:-4}
 MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200}
@@ -39,7 +43,8 @@ log() { echo "$(date '+%F %T') $*"; }
 # both, so every decision below is exercised against canned responses.
 do_api() { # do_api <path> [curl args...]
  local path=$1; shift
-  curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
+  # --fail-with-body: a refused create still prints why.
+  curl -sS --fail-with-body -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
    -H "Content-Type: application/json" "https://api.digitalocean.com/v2/$path" "$@"
 }
 gh_api() { # gh_api <path> [curl args...]
@@ -102,22 +107,59 @@ busy_runners() {
    | jq --arg l "$LABEL" '[.runners[] | select(.busy) | select(any(.labels[]; .name == $l))] | length'
 }

+# --- capacity --------------------------------------------------------------
+# "size region" lines to try, best first: SIZES order, then REGIONS order,
+# then every other region where DigitalOcean lists the size in stock.
+candidates() {
+  local page=1 response count catalog=""
+  while :; do
+    response=$(do_api "sizes?per_page=200&page=$page") || return 1
+    count=$(jq -er '.sizes | arrays | length' <<< "$response") || return 1
+    catalog+=$(jq -c '.sizes[]' <<< "$response")$'\n'
+    (( count == 200 )) || break
+    ((page += 1))
+  done
+  jq -rs --arg sizes "$SIZES" --arg regions "$REGIONS" '
+    ($regions | split(" ") | map(select(length > 0))) as $pref
+    | INDEX(.slug) as $by
+    | $sizes | split(" ") | map(select(length > 0)) | .[]
+    | . as $size | $by[$size] // {} | select(.available == true)
+    | .regions as $in
+    | (($pref | map(select(. as $r | $in | index($r)))) + ($in - $pref))[]
+    | "\($size) \(.)"' <<< "$catalog"
+}
+
 # --- create ----------------------------------------------------------------
+# Built once per tick by the first create; a refused pair is dropped from it.
+CANDIDATES=""
 create_droplet() {
-  local token userdata name body
+  local token userdata name size region body response
+  [[ -n $CANDIDATES ]] || CANDIDATES=$(candidates) || return 1
  token=$(gh_api "repos/$REPO/actions/runners/registration-token" -X POST | jq -r .token)
  userdata=$(sed -e "s|__REPO__|$REPO|g" -e "s|__RUNNER_TOKEN__|$token|g" \
                 -e "s|__RUNNER_LABELS__|$LABEL|g" -e "s|__RUNNER_VERSION__|$RUNNER_VERSION|g" \
                 -e "s|__SSH_KEYS_JSON__|$SSH_KEYS_JSON|" "$CLOUD_INIT")
  name="$TAG-$(date +%s)-$RANDOM"
-  body=$(jq -n --arg name "$name" --arg region "$REGION" --arg size "$SIZE" --arg image "$IMAGE" \
-    --arg tag "$TAG" --arg ud "$userdata" \
-    '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}')
-  log "creating $name ($SIZE)"
-  do_api droplets -X POST -d "$body" | jq -r '"created droplet \(.droplet.id)"'
+  while read -r size region; do
+    [[ -n $size ]] || continue
+    body=$(jq -n --arg name "$name" --arg region "$region" --arg size "$size" --arg image "$IMAGE" \
+      --arg tag "$TAG" --arg ud "$userdata" \
+      '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}')
+    log "creating $name ($size in $region)"
+    if response=$(do_api droplets -X POST -d "$body"); then
+      jq -r '"created droplet \(.droplet.id)"' <<< "$response"
+      return 0
+    fi
+    log "$size in $region refused: $(jq -r .message <<< "$response" 2>/dev/null || echo "$response")"
+    CANDIDATES=$(grep -Fvx "$size $region" <<< "$CANDIDATES" || true)
+  done <<< "$CANDIDATES"
+  # Every size refused everywhere: the rest of this tick's creates would be too.
+  log "no size in '$SIZES' can be created in any region"
+  return 1
 }

 controller_tick() {
+  CANDIDATES=""
  reap
  local queued live busy available need room
  queued=$(queued_jobs)
@@ -45,6 +45,10 @@ write_files:
    content: |
      #!/bin/bash
      set -euo pipefail
+      # Power off after the one job, and also when the download or the
+      # registration fails: the controller deletes powered-off droplets, but
+      # counts a running one as a runner still booting until MAX_AGE_MINUTES.
+      trap 'sudo poweroff' EXIT
      cd /home/runner
      mkdir -p actions-runner && cd actions-runner
      arch=$(uname -m); [[ $arch == x86_64 ]] && arch=x64
@@ -58,8 +62,6 @@ write_files:
        --labels "__RUNNER_LABELS__" \
        --replace
      ./run.sh
-      # One job done. Power off; the controller deletes powered-off droplets.
-      sudo poweroff

 runcmd:
  # With no account ssh key attached, DO expires root's password, and sshd
@@ -22,6 +22,7 @@ do_api() {
  local path=$1; shift
  echo "do $path $*" >>"$CALLS_FILE"
  case "$path" in
+    sizes\?*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;;
    droplets\?*) printf '%s\n' "$DROPLETS" | jq -Rs '{droplets: [split("\n")[] | select(length>0) | split(" ") | {id: .[0]|tonumber, status: .[1], created_at: .[2]}]}' ;;
    droplets) echo '{"droplet":{"id":999}}' ;;
    droplets/*) echo '{}' ;;
@@ -88,9 +89,43 @@ echo "PASS: API failures stop the queue query"

 # The create body must carry the tag (reaper scope) and substituted user-data.
 BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT
-do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; }
+do_api() {
+  case "$1" in
+    droplets) printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}' ;;
+    sizes*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;;
+    *) echo '{"droplets":[]}' ;;
+  esac
+}
 gh_api() { echo '{"token":"TOK"}'; }
-create_droplet >/dev/null
+CANDIDATES=""; create_droplet >/dev/null
 jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.user_data | test("--token \"TOK\"")) and (.user_data | test("__") | not)' "$BODY_FILE" >/dev/null \
  && echo "PASS: create body carries tag, size, substituted user-data" \
  || { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; }
+
+# Sizes are tried in SIZES order, each in the regions DigitalOcean lists it
+# in stock (REGIONS first); a 422 falls through to the next pair with the
+# refusal's message logged, and a refused pair is not retried in the tick.
+# When every pair is refused, the create fails.
+do_api() {
+  case "$1" in
+    sizes*) echo '{"sizes":[
+      {"slug":"small","available":true,"regions":["r1","r2"]},
+      {"slug":"gone","available":false,"regions":["r1"]},
+      {"slug":"big","available":true,"regions":["r3"]}]}' ;;
+    droplets)
+      local pair; pair=$(jq -r '"\(.size)@\(.region)"' <<< "${*: -1}"); echo "$pair" >>"$CALLS_FILE"
+      [[ $pair == big@r3 ]] && { echo '{"droplet":{"id":2}}'; return 0; }
+      echo '{"id":"unprocessable_entity","message":"Size is not available in this region."}'; return 22 ;;
+  esac
+}
+: >"$CALLS_FILE"; CANDIDATES=""
+out=$(SIZES="small gone big" REGIONS="r2"; create_droplet; create_droplet)
+[[ $(paste -sd' ' "$CALLS_FILE") == "small@r2 small@r1 big@r3 big@r3" \
+   && $out == *"small in r2 refused: Size is not available in this region."* && $out == *"created droplet 2"* ]] \
+  && echo "PASS: a refused size falls back to the next region, then the next size, logging why" \
+  || { echo "FAIL: size and region fallback"; echo "$out"; cat "$CALLS_FILE"; exit 1; }
+CANDIDATES=""
+if out=$(SIZES="small gone" create_droplet); then echo "FAIL: every pair refused looks like a create"; exit 1; fi
+[[ $out == *"no size in 'small gone' can be created in any region"* ]] \
+  && echo "PASS: every size refused everywhere fails the create" \
+  || { echo "FAIL: all-refused message"; echo "$out"; exit 1; }