diff --git a/ci/controller.sh b/ci/controller.sh index cc60950..b14f0e0 100755 --- a/ci/controller.sh +++ b/ci/controller.sh @@ -64,14 +64,33 @@ reap() { } # --- demand ---------------------------------------------------------------- +# Emit every item, including later pages of large build matrices. Keep API +# failures fatal so a failed query cannot look like an empty queue. +gh_items() { + local path=$1 key=$2 page=1 response count separator="?" + [[ $path != *"?"* ]] || separator="&" + while :; do + response=$(gh_api "${path}${separator}per_page=100&page=$page") || return 1 + count=$(jq -er --arg key "$key" '.[$key] | arrays | length' <<< "$response") || return 1 + jq -c --arg key "$key" '.[$key][]' <<< "$response" || return 1 + (( count == 100 )) || break + ((page += 1)) + done +} + queued_jobs() { - local run - gh_api "repos/$REPO/actions/runs?status=queued&per_page=50" --get \ - | jq -r '.workflow_runs[].id' | + local status runs run + # A workflow can be in progress while most of its matrix is still queued. + runs=$( + for status in queued in_progress; do + gh_items "repos/$REPO/actions/runs?status=$status" workflow_runs || exit 1 + done + ) || return 1 + jq -r '.id' <<< "$runs" | sort -u | while read -r run; do - gh_api "repos/$REPO/actions/runs/$run/jobs" \ - | jq -r --arg l "$LABEL" '.jobs[] | select(.status=="queued") | select(.labels | index($l)) | .id' - done | wc -l + gh_items "repos/$REPO/actions/runs/$run/jobs" jobs | + jq -r --arg l "$LABEL" 'select(.status=="queued") | select(.labels | index($l)) | .id' || return 1 + done | sort -u | wc -l } live_droplets() { diff --git a/tests/controller.sh b/tests/controller.sh index d777fc5..23a1bc2 100755 --- a/tests/controller.sh +++ b/tests/controller.sh @@ -31,8 +31,9 @@ gh_api() { local path=$1; shift echo "gh $path $*" >>"$CALLS_FILE" case "$path" in + */actions/runs\?status=in_progress*) echo '{"workflow_runs":[]}' ;; */actions/runs\?*) jq -nc --argjson n "$QUEUED" '{workflow_runs: [range($n) | {id: .}]}' ;; - */actions/runs/*/jobs) echo '{"jobs":[{"id":1,"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;; + */actions/runs/*/jobs\?*) echo '{"jobs":[{"id":'"$(echo "$path" | cut -d/ -f6)"',"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;; */actions/runners\?*) jq -nc --argjson n "$BUSY" '{runners: [range($n) | {busy: true, labels: [{name: "omarchy-builder"}]}]}' ;; */registration-token) echo '{"token":"T"}' ;; esac @@ -56,6 +57,35 @@ DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0 DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1 +# A large in-progress matrix has no waiting jobs on page one. The queued +# workflow is on page two of the run listing; the same run can appear in +# both status queries while GitHub updates it, so count its jobs once. +gh_api() { + case "$1" in + *runs?status=queued*page=1) jq -nc '{workflow_runs: [range(100) | {id: .}]}' ;; + *runs?status=queued*page=2) echo '{"workflow_runs":[{"id":999}]}' ;; + *runs?status=in_progress*) echo '{"workflow_runs":[{"id":999},{"id":1000}]}' ;; + *runs/999/jobs*page=1) jq -nc '{jobs: [range(100) | {id: .,status:"completed",labels:["omarchy-builder"]}]}' ;; + *runs/999/jobs*page=2) echo '{"jobs":[{"id":9991,"status":"queued","labels":["omarchy-builder"]}]}' ;; + *runs/1000/jobs*) echo '{"jobs":[{"id":10001,"status":"queued","labels":["omarchy-builder"]},{"id":10002,"status":"queued","labels":["ubuntu-latest"]},{"id":10003,"status":"in_progress","labels":["omarchy-builder"]}]}' ;; + *jobs*) echo '{"jobs":[]}' ;; + *) echo "Unexpected API request: $1" >&2; return 1 ;; + esac +} +[[ $(queued_jobs) == 2 ]] || { echo "FAIL: full queue across pages and workflow states"; exit 1; } +echo "PASS: later run/job pages and in-progress workflows count each waiting builder once" + +gh_api() { + case "$1" in + *runs?status=queued*) echo '{"workflow_runs":[{"id":1}]}' ;; + *runs?status=in_progress*) echo '{"workflow_runs":[]}' ;; + *jobs*page=1) jq -nc '{jobs:[range(100)|{id:.,status:"completed",labels:[]}]}' ;; + *) return 1 ;; + esac +} +if queued_jobs >/dev/null; then echo "FAIL: a failed later page looks like an empty queue"; exit 1; fi +echo "PASS: API failures stop the queue query" + # The create body must carry the tag (reaper scope) and substituted user-data. BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; }