Keep builders coming when DigitalOcean sells out a size or region (#861)

* Keep builders coming when DigitalOcean sells out a droplet size

ric1 sold out of g5-32vcpu-64gb-50gb, and every create came back 422. curl -f
dropped the reason and set -e ended the tick, so builders only appeared when
capacity happened to free up, and the journal showed nothing but "curl: (22)".

Try each of SIZES in turn, logging DigitalOcean's refusal message, and fail
the tick only when every size is refused. Builders now power off however
start.sh exits, so a failed registration is reaped instead of counting as a
booting runner until MAX_AGE_MINUTES. The controller unit pulls the checkout
before each tick, so merged controller fixes reach the box.

* Create builders in any region that has the size in stock

ric1 sold out of g5-32vcpu-128gb-50gb within minutes of the box switching to
it. Builders need nothing from a particular region, so read DigitalOcean's
size catalog once per tick and try each of SIZES in every region it lists in
stock, REGIONS first if set. A refused pair is dropped for the rest of the
tick.

tests/controller.sh

@@ -22,6 +22,7 @@ do_api() {
   local path=$1; shift
   echo "do $path $*" >>"$CALLS_FILE"
   case "$path" in
+    sizes\?*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;;
     droplets\?*) printf '%s\n' "$DROPLETS" | jq -Rs '{droplets: [split("\n")[] | select(length>0) | split(" ") | {id: .[0]|tonumber, status: .[1], created_at: .[2]}]}' ;;
     droplets) echo '{"droplet":{"id":999}}' ;;
     droplets/*) echo '{}' ;;
@@ -88,9 +89,43 @@ echo "PASS: API failures stop the queue query"
 
 # The create body must carry the tag (reaper scope) and substituted user-data.
 BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT
-do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; }
+do_api() {
+  case "$1" in
+    droplets) printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}' ;;
+    sizes*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;;
+    *) echo '{"droplets":[]}' ;;
+  esac
+}
 gh_api() { echo '{"token":"TOK"}'; }
-create_droplet >/dev/null
+CANDIDATES=""; create_droplet >/dev/null
 jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.user_data | test("--token \"TOK\"")) and (.user_data | test("__") | not)' "$BODY_FILE" >/dev/null \
   && echo "PASS: create body carries tag, size, substituted user-data" \
   || { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; }
+
+# Sizes are tried in SIZES order, each in the regions DigitalOcean lists it
+# in stock (REGIONS first); a 422 falls through to the next pair with the
+# refusal's message logged, and a refused pair is not retried in the tick.
+# When every pair is refused, the create fails.
+do_api() {
+  case "$1" in
+    sizes*) echo '{"sizes":[
+      {"slug":"small","available":true,"regions":["r1","r2"]},
+      {"slug":"gone","available":false,"regions":["r1"]},
+      {"slug":"big","available":true,"regions":["r3"]}]}' ;;
+    droplets)
+      local pair; pair=$(jq -r '"\(.size)@\(.region)"' <<< "${*: -1}"); echo "$pair" >>"$CALLS_FILE"
+      [[ $pair == big@r3 ]] && { echo '{"droplet":{"id":2}}'; return 0; }
+      echo '{"id":"unprocessable_entity","message":"Size is not available in this region."}'; return 22 ;;
+  esac
+}
+: >"$CALLS_FILE"; CANDIDATES=""
+out=$(SIZES="small gone big" REGIONS="r2"; create_droplet; create_droplet)
+[[ $(paste -sd' ' "$CALLS_FILE") == "small@r2 small@r1 big@r3 big@r3" \
+   && $out == *"small in r2 refused: Size is not available in this region."* && $out == *"created droplet 2"* ]] \
+  && echo "PASS: a refused size falls back to the next region, then the next size, logging why" \
+  || { echo "FAIL: size and region fallback"; echo "$out"; cat "$CALLS_FILE"; exit 1; }
+CANDIDATES=""
+if out=$(SIZES="small gone" create_droplet); then echo "FAIL: every pair refused looks like a create"; exit 1; fi
+[[ $out == *"no size in 'small gone' can be created in any region"* ]] \
+  && echo "PASS: every size refused everywhere fails the create" \
+  || { echo "FAIL: all-refused message"; echo "$out"; exit 1; }