Replace builder droplets stuck provisioning (#864)
A droplet DigitalOcean still reports as "new" never registers a runner, but the controller counted it as one booting and created no replacement until MAX_AGE_MINUTES. A publish job sat queued for minutes behind one in mkc1. Delete droplets still provisioning after MAX_BOOT_MINUTES (10) and leave them out of the live count, so the same tick creates a replacement.
This commit is contained in:
1 parent
259a3fa8b8
commit
5961a3ff5d
4 files changed
+18
-4
No files matched your search
+1
-1
@@ -14,7 +14,7 @@ signing on merge exactly as before.
|
|||||||
- `controller.sh` — systemd timer every minute on a small always-on droplet.
|
- `controller.sh` — systemd timer every minute on a small always-on droplet.
|
||||||
Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up
|
Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet per job up
|
||||||
to `MAX_DROPLETS`, deletes droplets that are powered off or older than
|
to `MAX_DROPLETS`, deletes droplets that are powered off or older than
|
||||||
`MAX_AGE_MINUTES`. Builders go in any region DigitalOcean lists the size in
|
`MAX_AGE_MINUTES`, or still provisioning after `MAX_BOOT_MINUTES`. Builders go in any region DigitalOcean lists the size in
|
||||||
stock in (`REGIONS` only sets which to try first); a refused create, logged
|
stock in (`REGIONS` only sets which to try first); a refused create, logged
|
||||||
with DigitalOcean's message, falls back to the next region, then the next
|
with DigitalOcean's message, falls back to the next region, then the next
|
||||||
of `SIZES`. No inbound endpoint. Plain curl against both APIs, no
|
of `SIZES`. No inbound endpoint. Plain curl against both APIs, no
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ SIZES="g5-32vcpu-64gb-50gb g5-32vcpu-128gb-50gb"
|
|||||||
REGIONS=
|
REGIONS=
|
||||||
MAX_DROPLETS=6
|
MAX_DROPLETS=6
|
||||||
MAX_AGE_MINUTES=200
|
MAX_AGE_MINUTES=200
|
||||||
|
# Delete a droplet DigitalOcean still reports as provisioning after this long.
|
||||||
|
MAX_BOOT_MINUTES=10
|
||||||
LOCK=/run/omarchy-controller/lock
|
LOCK=/run/omarchy-controller/lock
|
||||||
# Operator public keys for root on every builder droplet (JSON array).
|
# Operator public keys for root on every builder droplet (JSON array).
|
||||||
# create.sh fills this from the operators' GitHub keys.
|
# create.sh fills this from the operators' GitHub keys.
|
||||||
|
|||||||
+12
-3
@@ -4,7 +4,8 @@
|
|||||||
# Run from a systemd timer every minute on a small always-on droplet. No
|
# Run from a systemd timer every minute on a small always-on droplet. No
|
||||||
# inbound endpoint: it polls GitHub for queued jobs wanting our label, creates
|
# inbound endpoint: it polls GitHub for queued jobs wanting our label, creates
|
||||||
# one ephemeral droplet per job (up to MAX_DROPLETS), and deletes droplets
|
# one ephemeral droplet per job (up to MAX_DROPLETS), and deletes droplets
|
||||||
# that have powered off or exceeded MAX_AGE_MINUTES. The reaper does not
|
# that have powered off, exceeded MAX_AGE_MINUTES, or are still provisioning
|
||||||
|
# after MAX_BOOT_MINUTES. The reaper does not
|
||||||
# trust its own bookkeeping: it lists by tag and acts on what DigitalOcean
|
# trust its own bookkeeping: it lists by tag and acts on what DigitalOcean
|
||||||
# reports.
|
# reports.
|
||||||
#
|
#
|
||||||
@@ -30,6 +31,10 @@ REGIONS=${REGIONS:-${REGION:-}}
|
|||||||
IMAGE=${IMAGE:-ubuntu-24-04-x64}
|
IMAGE=${IMAGE:-ubuntu-24-04-x64}
|
||||||
MAX_DROPLETS=${MAX_DROPLETS:-4}
|
MAX_DROPLETS=${MAX_DROPLETS:-4}
|
||||||
MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200}
|
MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200}
|
||||||
|
# A droplet DigitalOcean still reports as "new" this long after creation is
|
||||||
|
# stuck provisioning. Left alone it counts as a runner booting, and holds a
|
||||||
|
# queued job until MAX_AGE_MINUTES.
|
||||||
|
MAX_BOOT_MINUTES=${MAX_BOOT_MINUTES:-10}
|
||||||
RUNNER_VERSION=${RUNNER_VERSION:-2.337.0}
|
RUNNER_VERSION=${RUNNER_VERSION:-2.337.0}
|
||||||
CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml}
|
CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml}
|
||||||
# Operator public keys authorized on every builder (JSON array of strings).
|
# Operator public keys authorized on every builder (JSON array of strings).
|
||||||
@@ -60,7 +65,8 @@ reap() {
|
|||||||
while read -r id status created; do
|
while read -r id status created; do
|
||||||
[[ -n "$id" ]] || continue
|
[[ -n "$id" ]] || continue
|
||||||
age=$(( (now - $(date -d "$created" +%s)) / 60 ))
|
age=$(( (now - $(date -d "$created" +%s)) / 60 ))
|
||||||
if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )); then
|
if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )) ||
|
||||||
|
{ [[ $status == new ]] && (( age > MAX_BOOT_MINUTES )); }; then
|
||||||
log "deleting droplet $id (status=$status age=${age}m)"
|
log "deleting droplet $id (status=$status age=${age}m)"
|
||||||
do_api "droplets/$id" -X DELETE
|
do_api "droplets/$id" -X DELETE
|
||||||
fi
|
fi
|
||||||
@@ -99,7 +105,10 @@ queued_jobs() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
live_droplets() {
|
live_droplets() {
|
||||||
do_api "droplets?tag_name=$TAG&per_page=200" | jq '[.droplets[] | select(.status != "off")] | length'
|
# Not the ones reap() just deleted: DigitalOcean can list them for a while.
|
||||||
|
do_api "droplets?tag_name=$TAG&per_page=200" | jq --argjson boot "$MAX_BOOT_MINUTES" '
|
||||||
|
[.droplets[] | select(.status != "off")
|
||||||
|
| select(.status != "new" or (now - (.created_at | fromdateiso8601)) / 60 <= $boot)] | length'
|
||||||
}
|
}
|
||||||
|
|
||||||
busy_runners() {
|
busy_runners() {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ CONTROLLER_LIBRARY_ONLY=1 source "$ROOT/ci/controller.sh"
|
|||||||
CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT
|
CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT
|
||||||
NOW=$(date -u +%FT%TZ)
|
NOW=$(date -u +%FT%TZ)
|
||||||
OLD=$(date -u -d '5 hours ago' +%FT%TZ)
|
OLD=$(date -u -d '5 hours ago' +%FT%TZ)
|
||||||
|
STUCK=$(date -u -d '15 minutes ago' +%FT%TZ)
|
||||||
|
|
||||||
# Scenario state: DROPLETS is "id status created" lines, QUEUED a count,
|
# Scenario state: DROPLETS is "id status created" lines, QUEUED a count,
|
||||||
# BUSY a count.
|
# BUSY a count.
|
||||||
@@ -54,6 +55,8 @@ DROPLETS="1 active $NOW" QUEUED=1 BUSY=1; run; check "one queued, one live but b
|
|||||||
DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0
|
DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0
|
||||||
DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1
|
DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1
|
||||||
DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1
|
DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1
|
||||||
|
DROPLETS="1 new $STUCK" QUEUED=1 BUSY=0; run; check "droplet stuck provisioning reaped and replaced" 1 1
|
||||||
|
DROPLETS="1 new $NOW" QUEUED=1 BUSY=0; run; check "droplet still provisioning is left to boot" 0 0
|
||||||
DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0
|
DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0
|
||||||
DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0
|
DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0
|
||||||
DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1
|
DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1
|
||||||
|
|||||||
Reference in new issue
Block a user