Keep builders coming when DigitalOcean sells out a size or region (#861)
* Keep builders coming when DigitalOcean sells out a droplet size ric1 sold out of g5-32vcpu-64gb-50gb, and every create came back 422. curl -f dropped the reason and set -e ended the tick, so builders only appeared when capacity happened to free up, and the journal showed nothing but "curl: (22)". Try each of SIZES in turn, logging DigitalOcean's refusal message, and fail the tick only when every size is refused. Builders now power off however start.sh exits, so a failed registration is reaped instead of counting as a booting runner until MAX_AGE_MINUTES. The controller unit pulls the checkout before each tick, so merged controller fixes reach the box. * Create builders in any region that has the size in stock ric1 sold out of g5-32vcpu-128gb-50gb within minutes of the box switching to it. Builders need nothing from a particular region, so read DigitalOcean's size catalog once per tick and try each of SIZES in every region it lists in stock, REGIONS first if set. A refused pair is dropped for the rest of the tick.
This commit is contained in:
Ryan Hughes authored and GitHub committed
1 parent
024943141d
commit
128c5647ba
6 files changed
+111
-17
No files matched your search
+12
-2
@@ -12,9 +12,12 @@ signing on merge exactly as before.
|
||||
- `runner-cloud-init.yaml` — Ubuntu 24.04 user-data: docker + buildx, the
|
||||
GitHub runner registered `--ephemeral`, runs one job, powers off.
|
||||
- `controller.sh` — systemd timer every minute on a small always-on droplet.
|
||||
Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet (ric1) per job up
|
||||
Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up
|
||||
to `MAX_DROPLETS`, deletes droplets that are powered off or older than
|
||||
`MAX_AGE_MINUTES`. No inbound endpoint. Plain curl against both APIs, no
|
||||
`MAX_AGE_MINUTES`. Builders go in any region DigitalOcean lists the size in
|
||||
stock in (`REGIONS` only sets which to try first); a refused create, logged
|
||||
with DigitalOcean's message, falls back to the next region, then the next
|
||||
of `SIZES`. No inbound endpoint. Plain curl against both APIs, no
|
||||
doctl and no gh: a token in the environment cannot pick the wrong account
|
||||
the way a saved doctl context can. Needs curl and jq.
|
||||
`tests/controller.sh` exercises every decision against canned responses.
|
||||
@@ -31,6 +34,13 @@ Administration read+write (registration tokens). The DO token is baked into
|
||||
the box's env file, so it is the account that pays for builder droplets.
|
||||
Watch it with `journalctl -u omarchy-controller -f` on the box.
|
||||
|
||||
Each tick pulls the box's checkout first, so a merged `controller.sh` is live
|
||||
within a minute. The unit and timer are copies made at creation; after
|
||||
changing them, on the box:
|
||||
|
||||
cp /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.{service,timer} /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
|
||||
## What the spike proved (2026-09-17, fork ryanrhughes/omarchy-pkgs)
|
||||
|
||||
- `bin/build` works from a bare clone: with no local published tree it
|
||||
|
||||
@@ -5,8 +5,10 @@ GITHUB_TOKEN=github_pat_...
|
||||
REPO=omacom/omarchy-pkgs
|
||||
LABEL=omarchy-builder
|
||||
TAG=omarchy-builder
|
||||
REGION=ric1
|
||||
SIZE=g5-32vcpu-64gb-50gb
|
||||
# Builder sizes, tried in order in any region that has them in stock.
|
||||
SIZES="g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb"
|
||||
# Optional: regions to try first, e.g. "ric1". Empty means any.
|
||||
REGIONS=
|
||||
MAX_DROPLETS=6
|
||||
MAX_AGE_MINUTES=200
|
||||
LOCK=/run/omarchy-controller/lock
|
||||
|
||||
@@ -7,6 +7,9 @@ Wants=network-online.target
|
||||
Type=oneshot
|
||||
User=controller
|
||||
EnvironmentFile=/etc/omarchy-controller.env
|
||||
# Run the branch's current controller, not the one cloned when the box was
|
||||
# built. As root (the checkout's owner); a failed pull keeps the last one.
|
||||
ExecStartPre=-+/usr/bin/git -C /opt/omarchy-pkgs pull --ff-only --quiet
|
||||
ExecStart=/opt/omarchy-pkgs/ci/controller.sh
|
||||
# The reaper's safety net is time, not state; a hung tick must not hold the lock.
|
||||
TimeoutStartSec=240
|
||||
+51
-9
@@ -21,8 +21,12 @@ REPO=${REPO:?owner/name}
|
||||
: "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}"
|
||||
LABEL=${LABEL:-omarchy-builder}
|
||||
TAG=${TAG:-omarchy-builder}
|
||||
REGION=${REGION:-ric1}
|
||||
SIZE=${SIZE:-g5-32vcpu-64gb-50gb}
|
||||
# Sizes to try, in order, in any region DigitalOcean lists them in stock. A
|
||||
# size can sell out in a region for hours; the create is then refused with
|
||||
# 422 and the next region, then the next size, is tried. REGIONS only orders
|
||||
# the regions tried first. SIZE and REGION, if set, are one-item lists.
|
||||
SIZES=${SIZES:-${SIZE:-g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb}}
|
||||
REGIONS=${REGIONS:-${REGION:-}}
|
||||
IMAGE=${IMAGE:-ubuntu-24-04-x64}
|
||||
MAX_DROPLETS=${MAX_DROPLETS:-4}
|
||||
MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200}
|
||||
@@ -39,7 +43,8 @@ log() { echo "$(date '+%F %T') $*"; }
|
||||
# both, so every decision below is exercised against canned responses.
|
||||
do_api() { # do_api <path> [curl args...]
|
||||
local path=$1; shift
|
||||
curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
|
||||
# --fail-with-body: a refused create still prints why.
|
||||
curl -sS --fail-with-body -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
|
||||
-H "Content-Type: application/json" "https://api.digitalocean.com/v2/$path" "$@"
|
||||
}
|
||||
gh_api() { # gh_api <path> [curl args...]
|
||||
@@ -102,22 +107,59 @@ busy_runners() {
|
||||
| jq --arg l "$LABEL" '[.runners[] | select(.busy) | select(any(.labels[]; .name == $l))] | length'
|
||||
}
|
||||
|
||||
# --- capacity --------------------------------------------------------------
|
||||
# "size region" lines to try, best first: SIZES order, then REGIONS order,
|
||||
# then every other region where DigitalOcean lists the size in stock.
|
||||
candidates() {
|
||||
local page=1 response count catalog=""
|
||||
while :; do
|
||||
response=$(do_api "sizes?per_page=200&page=$page") || return 1
|
||||
count=$(jq -er '.sizes | arrays | length' <<< "$response") || return 1
|
||||
catalog+=$(jq -c '.sizes[]' <<< "$response")$'\n'
|
||||
(( count == 200 )) || break
|
||||
((page += 1))
|
||||
done
|
||||
jq -rs --arg sizes "$SIZES" --arg regions "$REGIONS" '
|
||||
($regions | split(" ") | map(select(length > 0))) as $pref
|
||||
| INDEX(.slug) as $by
|
||||
| $sizes | split(" ") | map(select(length > 0)) | .[]
|
||||
| . as $size | $by[$size] // {} | select(.available == true)
|
||||
| .regions as $in
|
||||
| (($pref | map(select(. as $r | $in | index($r)))) + ($in - $pref))[]
|
||||
| "\($size) \(.)"' <<< "$catalog"
|
||||
}
|
||||
|
||||
# --- create ----------------------------------------------------------------
|
||||
# Built once per tick by the first create; a refused pair is dropped from it.
|
||||
CANDIDATES=""
|
||||
create_droplet() {
|
||||
local token userdata name body
|
||||
local token userdata name size region body response
|
||||
[[ -n $CANDIDATES ]] || CANDIDATES=$(candidates) || return 1
|
||||
token=$(gh_api "repos/$REPO/actions/runners/registration-token" -X POST | jq -r .token)
|
||||
userdata=$(sed -e "s|__REPO__|$REPO|g" -e "s|__RUNNER_TOKEN__|$token|g" \
|
||||
-e "s|__RUNNER_LABELS__|$LABEL|g" -e "s|__RUNNER_VERSION__|$RUNNER_VERSION|g" \
|
||||
-e "s|__SSH_KEYS_JSON__|$SSH_KEYS_JSON|" "$CLOUD_INIT")
|
||||
name="$TAG-$(date +%s)-$RANDOM"
|
||||
body=$(jq -n --arg name "$name" --arg region "$REGION" --arg size "$SIZE" --arg image "$IMAGE" \
|
||||
--arg tag "$TAG" --arg ud "$userdata" \
|
||||
'{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}')
|
||||
log "creating $name ($SIZE)"
|
||||
do_api droplets -X POST -d "$body" | jq -r '"created droplet \(.droplet.id)"'
|
||||
while read -r size region; do
|
||||
[[ -n $size ]] || continue
|
||||
body=$(jq -n --arg name "$name" --arg region "$region" --arg size "$size" --arg image "$IMAGE" \
|
||||
--arg tag "$TAG" --arg ud "$userdata" \
|
||||
'{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}')
|
||||
log "creating $name ($size in $region)"
|
||||
if response=$(do_api droplets -X POST -d "$body"); then
|
||||
jq -r '"created droplet \(.droplet.id)"' <<< "$response"
|
||||
return 0
|
||||
fi
|
||||
log "$size in $region refused: $(jq -r .message <<< "$response" 2>/dev/null || echo "$response")"
|
||||
CANDIDATES=$(grep -Fvx "$size $region" <<< "$CANDIDATES" || true)
|
||||
done <<< "$CANDIDATES"
|
||||
# Every size refused everywhere: the rest of this tick's creates would be too.
|
||||
log "no size in '$SIZES' can be created in any region"
|
||||
return 1
|
||||
}
|
||||
|
||||
controller_tick() {
|
||||
CANDIDATES=""
|
||||
reap
|
||||
local queued live busy available need room
|
||||
queued=$(queued_jobs)
|
||||
|
||||
@@ -45,6 +45,10 @@ write_files:
|
||||
content: |
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
# Power off after the one job, and also when the download or the
|
||||
# registration fails: the controller deletes powered-off droplets, but
|
||||
# counts a running one as a runner still booting until MAX_AGE_MINUTES.
|
||||
trap 'sudo poweroff' EXIT
|
||||
cd /home/runner
|
||||
mkdir -p actions-runner && cd actions-runner
|
||||
arch=$(uname -m); [[ $arch == x86_64 ]] && arch=x64
|
||||
@@ -58,8 +62,6 @@ write_files:
|
||||
--labels "__RUNNER_LABELS__" \
|
||||
--replace
|
||||
./run.sh
|
||||
# One job done. Power off; the controller deletes powered-off droplets.
|
||||
sudo poweroff
|
||||
|
||||
runcmd:
|
||||
# With no account ssh key attached, DO expires root's password, and sshd
|
||||
|
||||
Reference in new issue
Block a user