From 128c5647bad82e8c6c00901f4e9efb64da0fb47a Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Thu, 8 Oct 2026 10:50:06 -0400 Subject: [PATCH] Keep builders coming when DigitalOcean sells out a size or region (#861) * Keep builders coming when DigitalOcean sells out a droplet size ric1 sold out of g5-32vcpu-64gb-50gb, and every create came back 422. curl -f dropped the reason and set -e ended the tick, so builders only appeared when capacity happened to free up, and the journal showed nothing but "curl: (22)". Try each of SIZES in turn, logging DigitalOcean's refusal message, and fail the tick only when every size is refused. Builders now power off however start.sh exits, so a failed registration is reaped instead of counting as a booting runner until MAX_AGE_MINUTES. The controller unit pulls the checkout before each tick, so merged controller fixes reach the box. * Create builders in any region that has the size in stock ric1 sold out of g5-32vcpu-128gb-50gb within minutes of the box switching to it. Builders need nothing from a particular region, so read DigitalOcean's size catalog once per tick and try each of SIZES in every region it lists in stock, REGIONS first if set. A refused pair is dropped for the rest of the tick. --- ci/README.md | 14 ++++- ci/controller-box/controller.env.example | 6 +- ci/controller-box/omarchy-controller.service | 3 + ci/controller.sh | 60 +++++++++++++++++--- ci/runner-cloud-init.yaml | 6 +- tests/controller.sh | 39 ++++++++++++- 6 files changed, 111 insertions(+), 17 deletions(-) diff --git a/ci/README.md b/ci/README.md index 416f8d3..fb31448 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,9 +12,12 @@ signing on merge exactly as before. - `runner-cloud-init.yaml` — Ubuntu 24.04 user-data: docker + buildx, the GitHub runner registered `--ephemeral`, runs one job, powers off. - `controller.sh` — systemd timer every minute on a small always-on droplet. - Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet (ric1) per job up + Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up to `MAX_DROPLETS`, deletes droplets that are powered off or older than - `MAX_AGE_MINUTES`. No inbound endpoint. Plain curl against both APIs, no + `MAX_AGE_MINUTES`. Builders go in any region DigitalOcean lists the size in + stock in (`REGIONS` only sets which to try first); a refused create, logged + with DigitalOcean's message, falls back to the next region, then the next + of `SIZES`. No inbound endpoint. Plain curl against both APIs, no doctl and no gh: a token in the environment cannot pick the wrong account the way a saved doctl context can. Needs curl and jq. `tests/controller.sh` exercises every decision against canned responses. @@ -31,6 +34,13 @@ Administration read+write (registration tokens). The DO token is baked into the box's env file, so it is the account that pays for builder droplets. Watch it with `journalctl -u omarchy-controller -f` on the box. +Each tick pulls the box's checkout first, so a merged `controller.sh` is live +within a minute. The unit and timer are copies made at creation; after +changing them, on the box: + + cp /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.{service,timer} /etc/systemd/system/ + systemctl daemon-reload + ## What the spike proved (2026-09-17, fork ryanrhughes/omarchy-pkgs) - `bin/build` works from a bare clone: with no local published tree it diff --git a/ci/controller-box/controller.env.example b/ci/controller-box/controller.env.example index 37ae372..669e47d 100644 --- a/ci/controller-box/controller.env.example +++ b/ci/controller-box/controller.env.example @@ -5,8 +5,10 @@ GITHUB_TOKEN=github_pat_... REPO=omacom/omarchy-pkgs LABEL=omarchy-builder TAG=omarchy-builder -REGION=ric1 -SIZE=g5-32vcpu-64gb-50gb +# Builder sizes, tried in order in any region that has them in stock. +SIZES="g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb" +# Optional: regions to try first, e.g. "ric1". Empty means any. +REGIONS= MAX_DROPLETS=6 MAX_AGE_MINUTES=200 LOCK=/run/omarchy-controller/lock diff --git a/ci/controller-box/omarchy-controller.service b/ci/controller-box/omarchy-controller.service index 12d2e9c..e8848b8 100644 --- a/ci/controller-box/omarchy-controller.service +++ b/ci/controller-box/omarchy-controller.service @@ -7,6 +7,9 @@ Wants=network-online.target Type=oneshot User=controller EnvironmentFile=/etc/omarchy-controller.env +# Run the branch's current controller, not the one cloned when the box was +# built. As root (the checkout's owner); a failed pull keeps the last one. +ExecStartPre=-+/usr/bin/git -C /opt/omarchy-pkgs pull --ff-only --quiet ExecStart=/opt/omarchy-pkgs/ci/controller.sh # The reaper's safety net is time, not state; a hung tick must not hold the lock. TimeoutStartSec=240 diff --git a/ci/controller.sh b/ci/controller.sh index b14f0e0..60b15b1 100755 --- a/ci/controller.sh +++ b/ci/controller.sh @@ -21,8 +21,12 @@ REPO=${REPO:?owner/name} : "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}" LABEL=${LABEL:-omarchy-builder} TAG=${TAG:-omarchy-builder} -REGION=${REGION:-ric1} -SIZE=${SIZE:-g5-32vcpu-64gb-50gb} +# Sizes to try, in order, in any region DigitalOcean lists them in stock. A +# size can sell out in a region for hours; the create is then refused with +# 422 and the next region, then the next size, is tried. REGIONS only orders +# the regions tried first. SIZE and REGION, if set, are one-item lists. +SIZES=${SIZES:-${SIZE:-g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb}} +REGIONS=${REGIONS:-${REGION:-}} IMAGE=${IMAGE:-ubuntu-24-04-x64} MAX_DROPLETS=${MAX_DROPLETS:-4} MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200} @@ -39,7 +43,8 @@ log() { echo "$(date '+%F %T') $*"; } # both, so every decision below is exercised against canned responses. do_api() { # do_api [curl args...] local path=$1; shift - curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ + # --fail-with-body: a refused create still prints why. + curl -sS --fail-with-body -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ -H "Content-Type: application/json" "https://api.digitalocean.com/v2/$path" "$@" } gh_api() { # gh_api [curl args...] @@ -102,22 +107,59 @@ busy_runners() { | jq --arg l "$LABEL" '[.runners[] | select(.busy) | select(any(.labels[]; .name == $l))] | length' } +# --- capacity -------------------------------------------------------------- +# "size region" lines to try, best first: SIZES order, then REGIONS order, +# then every other region where DigitalOcean lists the size in stock. +candidates() { + local page=1 response count catalog="" + while :; do + response=$(do_api "sizes?per_page=200&page=$page") || return 1 + count=$(jq -er '.sizes | arrays | length' <<< "$response") || return 1 + catalog+=$(jq -c '.sizes[]' <<< "$response")$'\n' + (( count == 200 )) || break + ((page += 1)) + done + jq -rs --arg sizes "$SIZES" --arg regions "$REGIONS" ' + ($regions | split(" ") | map(select(length > 0))) as $pref + | INDEX(.slug) as $by + | $sizes | split(" ") | map(select(length > 0)) | .[] + | . as $size | $by[$size] // {} | select(.available == true) + | .regions as $in + | (($pref | map(select(. as $r | $in | index($r)))) + ($in - $pref))[] + | "\($size) \(.)"' <<< "$catalog" +} + # --- create ---------------------------------------------------------------- +# Built once per tick by the first create; a refused pair is dropped from it. +CANDIDATES="" create_droplet() { - local token userdata name body + local token userdata name size region body response + [[ -n $CANDIDATES ]] || CANDIDATES=$(candidates) || return 1 token=$(gh_api "repos/$REPO/actions/runners/registration-token" -X POST | jq -r .token) userdata=$(sed -e "s|__REPO__|$REPO|g" -e "s|__RUNNER_TOKEN__|$token|g" \ -e "s|__RUNNER_LABELS__|$LABEL|g" -e "s|__RUNNER_VERSION__|$RUNNER_VERSION|g" \ -e "s|__SSH_KEYS_JSON__|$SSH_KEYS_JSON|" "$CLOUD_INIT") name="$TAG-$(date +%s)-$RANDOM" - body=$(jq -n --arg name "$name" --arg region "$REGION" --arg size "$SIZE" --arg image "$IMAGE" \ - --arg tag "$TAG" --arg ud "$userdata" \ - '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}') - log "creating $name ($SIZE)" - do_api droplets -X POST -d "$body" | jq -r '"created droplet \(.droplet.id)"' + while read -r size region; do + [[ -n $size ]] || continue + body=$(jq -n --arg name "$name" --arg region "$region" --arg size "$size" --arg image "$IMAGE" \ + --arg tag "$TAG" --arg ud "$userdata" \ + '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}') + log "creating $name ($size in $region)" + if response=$(do_api droplets -X POST -d "$body"); then + jq -r '"created droplet \(.droplet.id)"' <<< "$response" + return 0 + fi + log "$size in $region refused: $(jq -r .message <<< "$response" 2>/dev/null || echo "$response")" + CANDIDATES=$(grep -Fvx "$size $region" <<< "$CANDIDATES" || true) + done <<< "$CANDIDATES" + # Every size refused everywhere: the rest of this tick's creates would be too. + log "no size in '$SIZES' can be created in any region" + return 1 } controller_tick() { + CANDIDATES="" reap local queued live busy available need room queued=$(queued_jobs) diff --git a/ci/runner-cloud-init.yaml b/ci/runner-cloud-init.yaml index c05e32e..1cb409e 100644 --- a/ci/runner-cloud-init.yaml +++ b/ci/runner-cloud-init.yaml @@ -45,6 +45,10 @@ write_files: content: | #!/bin/bash set -euo pipefail + # Power off after the one job, and also when the download or the + # registration fails: the controller deletes powered-off droplets, but + # counts a running one as a runner still booting until MAX_AGE_MINUTES. + trap 'sudo poweroff' EXIT cd /home/runner mkdir -p actions-runner && cd actions-runner arch=$(uname -m); [[ $arch == x86_64 ]] && arch=x64 @@ -58,8 +62,6 @@ write_files: --labels "__RUNNER_LABELS__" \ --replace ./run.sh - # One job done. Power off; the controller deletes powered-off droplets. - sudo poweroff runcmd: # With no account ssh key attached, DO expires root's password, and sshd diff --git a/tests/controller.sh b/tests/controller.sh index 23a1bc2..c02c0db 100755 --- a/tests/controller.sh +++ b/tests/controller.sh @@ -22,6 +22,7 @@ do_api() { local path=$1; shift echo "do $path $*" >>"$CALLS_FILE" case "$path" in + sizes\?*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;; droplets\?*) printf '%s\n' "$DROPLETS" | jq -Rs '{droplets: [split("\n")[] | select(length>0) | split(" ") | {id: .[0]|tonumber, status: .[1], created_at: .[2]}]}' ;; droplets) echo '{"droplet":{"id":999}}' ;; droplets/*) echo '{}' ;; @@ -88,9 +89,43 @@ echo "PASS: API failures stop the queue query" # The create body must carry the tag (reaper scope) and substituted user-data. BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT -do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; } +do_api() { + case "$1" in + droplets) printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}' ;; + sizes*) echo '{"sizes":[{"slug":"g5-32vcpu-64gb-50gb","available":true,"regions":["ric1"]}]}' ;; + *) echo '{"droplets":[]}' ;; + esac +} gh_api() { echo '{"token":"TOK"}'; } -create_droplet >/dev/null +CANDIDATES=""; create_droplet >/dev/null jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.user_data | test("--token \"TOK\"")) and (.user_data | test("__") | not)' "$BODY_FILE" >/dev/null \ && echo "PASS: create body carries tag, size, substituted user-data" \ || { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; } + +# Sizes are tried in SIZES order, each in the regions DigitalOcean lists it +# in stock (REGIONS first); a 422 falls through to the next pair with the +# refusal's message logged, and a refused pair is not retried in the tick. +# When every pair is refused, the create fails. +do_api() { + case "$1" in + sizes*) echo '{"sizes":[ + {"slug":"small","available":true,"regions":["r1","r2"]}, + {"slug":"gone","available":false,"regions":["r1"]}, + {"slug":"big","available":true,"regions":["r3"]}]}' ;; + droplets) + local pair; pair=$(jq -r '"\(.size)@\(.region)"' <<< "${*: -1}"); echo "$pair" >>"$CALLS_FILE" + [[ $pair == big@r3 ]] && { echo '{"droplet":{"id":2}}'; return 0; } + echo '{"id":"unprocessable_entity","message":"Size is not available in this region."}'; return 22 ;; + esac +} +: >"$CALLS_FILE"; CANDIDATES="" +out=$(SIZES="small gone big" REGIONS="r2"; create_droplet; create_droplet) +[[ $(paste -sd' ' "$CALLS_FILE") == "small@r2 small@r1 big@r3 big@r3" \ + && $out == *"small in r2 refused: Size is not available in this region."* && $out == *"created droplet 2"* ]] \ + && echo "PASS: a refused size falls back to the next region, then the next size, logging why" \ + || { echo "FAIL: size and region fallback"; echo "$out"; cat "$CALLS_FILE"; exit 1; } +CANDIDATES="" +if out=$(SIZES="small gone" create_droplet); then echo "FAIL: every pair refused looks like a create"; exit 1; fi +[[ $out == *"no size in 'small gone' can be created in any region"* ]] \ + && echo "PASS: every size refused everywhere fails the create" \ + || { echo "FAIL: all-refused message"; echo "$out"; exit 1; }