Count the full builder queue across workflow states (#859)

ci/controller.sh

@@ -64,14 +64,33 @@ reap() {
 }
 
 # --- demand ----------------------------------------------------------------
+# Emit every item, including later pages of large build matrices. Keep API
+# failures fatal so a failed query cannot look like an empty queue.
+gh_items() {
+  local path=$1 key=$2 page=1 response count separator="?"
+  [[ $path != *"?"* ]] || separator="&"
+  while :; do
+    response=$(gh_api "${path}${separator}per_page=100&page=$page") || return 1
+    count=$(jq -er --arg key "$key" '.[$key] | arrays | length' <<< "$response") || return 1
+    jq -c --arg key "$key" '.[$key][]' <<< "$response" || return 1
+    (( count == 100 )) || break
+    ((page += 1))
+  done
+}
+
 queued_jobs() {
-  local run
-  gh_api "repos/$REPO/actions/runs?status=queued&per_page=50" --get \
-    | jq -r '.workflow_runs[].id' |
+  local status runs run
+  # A workflow can be in progress while most of its matrix is still queued.
+  runs=$(
+    for status in queued in_progress; do
+      gh_items "repos/$REPO/actions/runs?status=$status" workflow_runs || exit 1
+    done
+  ) || return 1
+  jq -r '.id' <<< "$runs" | sort -u |
   while read -r run; do
-    gh_api "repos/$REPO/actions/runs/$run/jobs" \
-      | jq -r --arg l "$LABEL" '.jobs[] | select(.status=="queued") | select(.labels | index($l)) | .id'
-  done | wc -l
+    gh_items "repos/$REPO/actions/runs/$run/jobs" jobs |
+      jq -r --arg l "$LABEL" 'select(.status=="queued") | select(.labels | index($l)) | .id' || return 1
+  done | sort -u | wc -l
 }
 
 live_droplets() {

tests/controller.sh

@@ -31,8 +31,9 @@ gh_api() {
   local path=$1; shift
   echo "gh $path $*" >>"$CALLS_FILE"
   case "$path" in
+    */actions/runs\?status=in_progress*) echo '{"workflow_runs":[]}' ;;
     */actions/runs\?*) jq -nc --argjson n "$QUEUED" '{workflow_runs: [range($n) | {id: .}]}' ;;
-    */actions/runs/*/jobs) echo '{"jobs":[{"id":1,"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;;
+    */actions/runs/*/jobs\?*) echo '{"jobs":[{"id":'"$(echo "$path" | cut -d/ -f6)"',"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;;
     */actions/runners\?*) jq -nc --argjson n "$BUSY" '{runners: [range($n) | {busy: true, labels: [{name: "omarchy-builder"}]}]}' ;;
     */registration-token) echo '{"token":"T"}' ;;
   esac
@@ -56,6 +57,35 @@ DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active
 DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0
 DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1
 
+# A large in-progress matrix has no waiting jobs on page one. The queued
+# workflow is on page two of the run listing; the same run can appear in
+# both status queries while GitHub updates it, so count its jobs once.
+gh_api() {
+  case "$1" in
+    *runs?status=queued*page=1) jq -nc '{workflow_runs: [range(100) | {id: .}]}' ;;
+    *runs?status=queued*page=2) echo '{"workflow_runs":[{"id":999}]}' ;;
+    *runs?status=in_progress*) echo '{"workflow_runs":[{"id":999},{"id":1000}]}' ;;
+    *runs/999/jobs*page=1) jq -nc '{jobs: [range(100) | {id: .,status:"completed",labels:["omarchy-builder"]}]}' ;;
+    *runs/999/jobs*page=2) echo '{"jobs":[{"id":9991,"status":"queued","labels":["omarchy-builder"]}]}' ;;
+    *runs/1000/jobs*) echo '{"jobs":[{"id":10001,"status":"queued","labels":["omarchy-builder"]},{"id":10002,"status":"queued","labels":["ubuntu-latest"]},{"id":10003,"status":"in_progress","labels":["omarchy-builder"]}]}' ;;
+    *jobs*) echo '{"jobs":[]}' ;;
+    *) echo "Unexpected API request: $1" >&2; return 1 ;;
+  esac
+}
+[[ $(queued_jobs) == 2 ]] || { echo "FAIL: full queue across pages and workflow states"; exit 1; }
+echo "PASS: later run/job pages and in-progress workflows count each waiting builder once"
+
+gh_api() {
+  case "$1" in
+    *runs?status=queued*) echo '{"workflow_runs":[{"id":1}]}' ;;
+    *runs?status=in_progress*) echo '{"workflow_runs":[]}' ;;
+    *jobs*page=1) jq -nc '{jobs:[range(100)|{id:.,status:"completed",labels:[]}]}' ;;
+    *) return 1 ;;
+  esac
+}
+if queued_jobs >/dev/null; then echo "FAIL: a failed later page looks like an empty queue"; exit 1; fi
+echo "PASS: API failures stop the queue query"
+
 # The create body must carry the tag (reaper scope) and substituted user-data.
 BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT
 do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; }