Merge pull request #663 from omacom/ci/sync-pr-churn

Keep sync PRs building across bot pushes

.github/scripts/approve-pr-workflows.cjs

@@ -1,7 +1,11 @@
 const BUILD = '.github/workflows/build-pr.yml';
 const TESTS = '.github/workflows/test.yml';
 
+// pullRequest/action/since default to the pull_request_target event. The
+// sync workflows pass them explicitly: GitHub creates no pull_request_target
+// run for a GITHUB_TOKEN push, so they release their own pushes' held runs.
 module.exports = async function approve({ github, context, core, vouchStatus,
+  pullRequest = context.payload.pull_request, action = context.payload.action, since,
   sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), attempts = 36 }) {
   // Missing/failed vouch lookups must not become approval. Denouncements
   // remain absolute, just as they are in the package build gate.
@@ -9,8 +13,8 @@ module.exports = async function approve({ github, context, core, vouchStatus,
     throw new Error(`Cannot approve workflows: vouch status is ${vouchStatus || 'missing'}.`);
   }
 
-  const expected = context.payload.pull_request;
-  const eventTime = Date.parse(expected.updated_at);
+  const expected = pullRequest;
+  const eventTime = Date.parse(since ?? expected.updated_at);
   if (!Number.isFinite(eventTime)) throw new Error('Missing PR event timestamp.');
   const approved = new Set();
   let precedingBuild;
@@ -47,7 +51,7 @@ module.exports = async function approve({ github, context, core, vouchStatus,
     const newestBuild = runs.findLast(run => run.path === BUILD);
     if (!newestBuild || !(Date.parse(newestBuild.created_at) >= eventTime) ||
         !runs.some(run => run.path === TESTS &&
-          (context.payload.action === 'labeled' || Date.parse(run.created_at) >= eventTime))) continue;
+          (action === 'labeled' || Date.parse(run.created_at) >= eventTime))) continue;
 
     if (precedingBuild) {
       const { data: run } = await github.rest.actions.getWorkflowRun({
@@ -71,7 +75,13 @@ module.exports = async function approve({ github, context, core, vouchStatus,
     await github.rest.actions.approveWorkflowRun({ ...context.repo, run_id: run.id });
     approved.add(run.id);
     core.info(`Approved ${run.path} run ${run.id} for PR #${expected.number}.`);
-    if (run.path === BUILD) precedingBuild = run.id;
+    // Only a newer held build needs this one to take the concurrency slot
+    // first. A lone build may sit pending behind an in-flight build of an
+    // older commit (sync branches queue rather than cancel); waiting for it
+    // to start would time out before the tests run was released.
+    if (run.path === BUILD && pending.some(other => other.path === BUILD && other.id > run.id)) {
+      precedingBuild = run.id;
+    }
     if (pending.length === 1) return;
   }
   throw new Error('Timed out waiting for PR workflows. Remove and reapply build-approved to retry.');

.github/scripts/approve-sync-push.cjs

@@ -0,0 +1,33 @@
+const approvePrWorkflows = require('./approve-pr-workflows.cjs');
+
+const BOT = 'github-actions[bot]';
+
+// A sync workflow pushes its branch with GITHUB_TOKEN. GitHub holds the
+// resulting pull_request runs for approval and, unlike a person's push,
+// creates no pull_request_target run, so approve-pr.yml never sees it. The
+// sync workflow therefore releases the runs for the commit it just pushed,
+// under the same rule approve-pr.yml applies: only while a maintainer's
+// build-approved label is on the PR. It acts only on its own bot-authored,
+// same-repository PR for the branch and commit it pushed.
+module.exports = async function approveSyncPush({ github, context, core,
+  number, branch, headSha, since, approve = approvePrWorkflows, ...options }) {
+  if (!Number.isInteger(number) || !branch || !headSha || !since) {
+    throw new Error('Missing sync PR number, branch, head SHA or push time.');
+  }
+  const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: number });
+  const repository = `${context.repo.owner}/${context.repo.repo}`;
+  if (pr.user?.login !== BOT || pr.head.repo?.full_name !== repository ||
+      pr.base.repo?.full_name !== repository || pr.head.ref !== branch) {
+    throw new Error(`PR #${number} is not ${BOT}'s ${branch} PR in ${repository}; refusing to approve.`);
+  }
+  if (pr.state !== 'open' || pr.head.sha !== headSha) {
+    core.info(`PR #${number} is closed or has moved past ${headSha}; nothing to approve.`);
+    return;
+  }
+  if (!pr.labels.some(label => label.name === 'build-approved')) {
+    core.info(`PR #${number} has no build-approved label; its runs wait for a maintainer.`);
+    return;
+  }
+  await approve({ github, context, core, vouchStatus: 'bot', pullRequest: pr,
+    action: 'synchronize', since, ...options });
+};

.github/scripts/sync-pr-branch.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Usage: sync-pr-branch.sh BASE_BRANCH [PACKAGE...]
+#
+# Prints the branch a sync workflow run pushes to, as branch=/scope= lines for
+# $GITHUB_OUTPUT. An unscoped (scheduled) run owns BASE_BRANCH and regenerates
+# it from master every time. A run scoped to named packages regenerates only
+# those, so it gets its own branch and PR: pushing it to BASE_BRANCH would
+# replace every other pending update there with just the named packages.
+set -euo pipefail
+
+base=${1:?base branch required}
+shift
+if (( $# == 0 )); then
+  printf 'branch=%s\nscope=\n' "$base"
+  exit 0
+fi
+
+names=()
+for name in "$@"; do
+  # Package directory names, as pacman allows them. Anything else is a typo
+  # or an attempt to smuggle something into a ref name or PR title.
+  if [[ ! $name =~ ^[a-z0-9@_+][a-z0-9@._+-]*$ ]]; then
+    echo "invalid package name: $name" >&2
+    exit 1
+  fi
+  names+=("$name")
+done
+mapfile -t names < <(printf '%s\n' "${names[@]}" | sort -u)
+
+scope="${names[*]}"
+slug=$(printf '%s\n' "${names[@]}" | sed 's/[^a-z0-9]\{1,\}/-/g; s/^-//; s/-$//' | paste -sd- -)
+# Keep long package lists to a readable ref; the hash keeps distinct lists apart.
+hash=$(printf '%s' "$scope" | sha256sum | cut -c1-10)
+if [[ -z $slug ]]; then
+  slug=$hash
+elif (( ${#slug} > 60 )); then
+  slug="${slug:0:48}"
+  slug="${slug%-}-$hash"
+fi
+printf 'branch=%s-%s\nscope=%s\n' "$base" "$slug" "$scope"

.github/workflows/build-pr.yml

@@ -20,9 +20,16 @@ on:
         description: "Space-separated package directories to build"
         required: true
 
+# A new push normally cancels the PR's in-flight build. The sync bots'
+# branches (auto/sync-*) are the exception: they are force-pushed with fresh
+# upstream releases several times a day, which kept cancelling multi-hour
+# aarch64 builds before they could finish. There the newest run waits
+# instead (GitHub keeps at most one pending run per group, replacing older
+# pending ones), and when it starts it reuses every artifact the finished
+# build uploaded, so only packages whose tree changed are built again.
 concurrency:
   group: build-${{ github.event.pull_request.number || github.run_id }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ !(github.event.pull_request.head.repo.full_name == github.repository && startsWith(github.head_ref, 'auto/sync-')) }}
 
 jobs:
   # Builds cost real machines, so they run only for trusted authors:

.github/workflows/sync-rebuilds.yml

@@ -17,6 +17,12 @@ jobs:
     permissions:
       contents: write
       pull-requests: write
+    outputs:
+      branch: ${{ steps.branch.outputs.branch }}
+      pushed_at: ${{ steps.pushed.outputs.at }}
+      number: ${{ steps.cpr.outputs.pull-request-number }}
+      operation: ${{ steps.cpr.outputs.pull-request-operation }}
+      head_sha: ${{ steps.cpr.outputs.pull-request-head-sha }}
 
     steps:
       - name: Checkout repository
@@ -24,6 +30,17 @@ jobs:
         with:
           persist-credentials: false
 
+      # A scoped dispatch regenerates only the named packages. Pushed to the
+      # shared branch, that would replace every other pending update in its
+      # PR, so it gets a branch and PR of its own.
+      - name: Choose the PR branch
+        id: branch
+        env:
+          PACKAGES: ${{ github.event.inputs.packages }}
+        run: |
+          read -r -a package_args <<< "${PACKAGES:-}"
+          .github/scripts/sync-pr-branch.sh auto/sync-rebuilds "${package_args[@]}" | tee -a "$GITHUB_OUTPUT"
+
       # Runs in an Arch container against the mirror the x86_64 builder itself
       # uses, because the question being asked is what that builder will link
       # against and a different mirror can be hours ahead of it. Recording a
@@ -68,13 +85,21 @@ jobs:
             echo "has_changes=true" >> "$GITHUB_OUTPUT"
           fi
 
+      # Runs created by this push are newer than this; the approve job
+      # waits for them. A minute's slack absorbs runner clock skew.
+      - name: Record push time
+        if: steps.changes.outputs.has_changes == 'true'
+        id: pushed
+        run: echo "at=$(date -u -d '-1 minute' +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
+
       - name: Create Pull Request
         if: steps.changes.outputs.has_changes == 'true'
+        id: cpr
         uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: 'chore: rebuild against updated dependencies'
-          title: 'chore: rebuild against updated dependencies'
+          title: "chore: rebuild against updated dependencies${{ steps.branch.outputs.scope && format(' ({0})', steps.branch.outputs.scope) || '' }}"
           body: |
             Automated pkgrel bump for packages that link against a dependency
             which has moved in the official repositories.
@@ -84,7 +109,7 @@ jobs:
             bump is what makes the rebuilt package an upgrade pacman will offer;
             without it the build produces the version already published and no
             one receives it.
-          branch: auto/sync-rebuilds
+          branch: ${{ steps.branch.outputs.branch }}
           delete-branch: true
           labels: automated
           reviewers: ryanrhughes
@@ -100,3 +125,35 @@ jobs:
               "🔴 <strong>Rebuild trigger sync failed</strong><br><a href=\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\">View run</a>" \
               '{content: $content}')" \
             "$BASECAMP_CHATBOT_URL"
+
+  # GitHub holds pull_request runs from a GITHUB_TOKEN push for approval and
+  # creates no pull_request_target run for it, so approve-pr.yml never sees
+  # the sync's own pushes. Once a maintainer has labelled the PR
+  # build-approved, release the held runs for the commit just pushed. A
+  # separate job, so the sync container's token never holds actions: write.
+  approve:
+    needs: sync
+    if: ${{ !cancelled() && needs.sync.outputs.operation == 'updated' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+      actions: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Release held build and test runs if build-approved
+        uses: actions/github-script@v7
+        env:
+          NUMBER: ${{ needs.sync.outputs.number }}
+          BRANCH: ${{ needs.sync.outputs.branch }}
+          HEAD_SHA: ${{ needs.sync.outputs.head_sha }}
+          SINCE: ${{ needs.sync.outputs.pushed_at }}
+        with:
+          script: |
+            const approve = require('./.github/scripts/approve-sync-push.cjs');
+            const { NUMBER, BRANCH, HEAD_SHA, SINCE } = process.env;
+            await approve({ github, context, core, number: Number(NUMBER),
+              branch: BRANCH, headSha: HEAD_SHA, since: SINCE });

.github/workflows/sync-upstream.yml

@@ -17,6 +17,12 @@ jobs:
     permissions:
       contents: write
       pull-requests: write
+    outputs:
+      branch: ${{ steps.branch.outputs.branch }}
+      pushed_at: ${{ steps.pushed.outputs.at }}
+      number: ${{ steps.cpr.outputs.pull-request-number }}
+      operation: ${{ steps.cpr.outputs.pull-request-operation }}
+      head_sha: ${{ steps.cpr.outputs.pull-request-head-sha }}
 
     steps:
       - name: Checkout repository
@@ -24,6 +30,17 @@ jobs:
         with:
           persist-credentials: false
 
+      # A scoped dispatch regenerates only the named packages. Pushed to the
+      # shared branch, that would replace every other pending update in its
+      # PR, so it gets a branch and PR of its own.
+      - name: Choose the PR branch
+        id: branch
+        env:
+          PACKAGES: ${{ github.event.inputs.packages }}
+        run: |
+          read -r -a package_args <<< "${PACKAGES:-}"
+          .github/scripts/sync-pr-branch.sh auto/sync-upstream "${package_args[@]}" | tee -a "$GITHUB_OUTPUT"
+
       # Runs in an Arch container for vercmp: whether a release is an upgrade has
       # to be decided by the same comparator pacman will use on users' machines.
       - name: Update packages from upstream release feeds
@@ -72,13 +89,21 @@ jobs:
             echo "has_changes=true" >> "$GITHUB_OUTPUT"
           fi
 
+      # Runs created by this push are newer than this; the approve job
+      # waits for them. A minute's slack absorbs runner clock skew.
+      - name: Record push time
+        if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
+        id: pushed
+        run: echo "at=$(date -u -d '-1 minute' +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
+
       - name: Create Pull Request
         if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
+        id: cpr
         uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: 'chore: sync upstream releases'
-          title: 'chore: sync upstream releases'
+          title: "chore: sync upstream releases${{ steps.branch.outputs.scope && format(' ({0})', steps.branch.outputs.scope) || '' }}"
           body: |
             Automated update of packages that track an upstream vendor release
             feed rather than the AUR.
@@ -86,7 +111,7 @@ jobs:
             Release watches and providers are declared in `.omarchy/package.json`;
             exceptional feeds use `.omarchy/upstream.sh`. Failed package updates
             are left untouched; check the workflow result for outstanding failures.
-          branch: auto/sync-upstream
+          branch: ${{ steps.branch.outputs.branch }}
           delete-branch: true
           labels: automated
           reviewers: ryanrhughes
@@ -102,3 +127,35 @@ jobs:
               "🔴 <strong>Upstream sync failed</strong><br><a href=\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\">View run</a>" \
               '{content: $content}')" \
             "$BASECAMP_CHATBOT_URL"
+
+  # GitHub holds pull_request runs from a GITHUB_TOKEN push for approval and
+  # creates no pull_request_target run for it, so approve-pr.yml never sees
+  # the sync's own pushes. Once a maintainer has labelled the PR
+  # build-approved, release the held runs for the commit just pushed. A
+  # separate job, so the sync container's token never holds actions: write.
+  approve:
+    needs: sync
+    if: ${{ !cancelled() && needs.sync.outputs.operation == 'updated' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: read
+      actions: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Release held build and test runs if build-approved
+        uses: actions/github-script@v7
+        env:
+          NUMBER: ${{ needs.sync.outputs.number }}
+          BRANCH: ${{ needs.sync.outputs.branch }}
+          HEAD_SHA: ${{ needs.sync.outputs.head_sha }}
+          SINCE: ${{ needs.sync.outputs.pushed_at }}
+        with:
+          script: |
+            const approve = require('./.github/scripts/approve-sync-push.cjs');
+            const { NUMBER, BRANCH, HEAD_SHA, SINCE } = process.env;
+            await approve({ github, context, core, number: Number(NUMBER),
+              branch: BRANCH, headSha: HEAD_SHA, since: SINCE });