Merge remote-tracking branch 'upstream/master' into spark/aarch64-settings-boot-dropins

.github/workflows/build-pr.yml

@@ -1,8 +1,9 @@
 name: Build changed packages
 
-# Build every package directory a PR touches, one job per package per arch, on
-# the self-hosted droplet pool. Artifacts are unsigned; publish.yml signs and
-# publishes them on merge.
+# Build every package directory a PR touches, one job per package per arch:
+# x86_64 on the self-hosted droplet pool, aarch64 natively on GitHub's arm64
+# runners. Artifacts are unsigned; publish.yml signs and publishes them on
+# merge.
 #
 # Tooling runs from the base branch; a PR supplies only pkgbuilds/. The
 # vouch gate limits who may spend compute; this limits what their PR can run.
@@ -87,8 +88,12 @@ jobs:
           if [[ -n "${{ github.event.inputs.packages }}" ]]; then
             names="${{ github.event.inputs.packages }}"
           else
+            # A package the PR deletes has nothing to build.
             names=$(git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" -- pkgbuilds \
-              | awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u)
+              | awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u \
+              | while read -r name; do
+                  if git cat-file -e "${{ github.event.pull_request.head.sha }}:pkgbuilds/$name" 2>/dev/null; then echo "$name"; fi
+                done)
           fi
           matrix=$(printf '%s\n' $names | bin/build-matrix)
           # A package directory whose exact tree already has a build artifact
@@ -159,13 +164,17 @@ jobs:
   build:
     needs: changes
     if: needs.changes.outputs.count != '0'
-    runs-on: [self-hosted, omarchy-builder]
+    # The droplets are x86, so aarch64 there runs under QEMU: omarchy-mac-boot
+    # took 2h47m of the 180 minutes, most of it in check().
+    # GitHub's arm64 runners (4 vCPU, 16 GB; ~100 GB disk free in our pilots)
+    # build it natively in 11 minutes, and linux-aurora in 30 (72 under QEMU).
+    runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || fromJSON('["self-hosted","omarchy-builder"]') }}
     timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.changes.outputs.matrix) }}
     steps:
-      # Tooling from base: everything that executes on this droplet's host
+      # Tooling from base: everything that executes on this runner's host
       # (bin/, helpers/, build/) comes from the base branch. Only the PR's
       # package directories are overlaid. A PR can therefore change what
       # gets built, never how the runner builds it. A PR that changes both
@@ -192,6 +201,19 @@ jobs:
         env:
           CONTAINER_ENGINE: docker
         run: bin/build --mirror edge --arch ${{ matrix.arch }} --package ${{ matrix.package }}
+      # makepkg's check() leaves meson's per-test output in the build tree,
+      # never on stdout, so a failing test shows only a summary line in this
+      # job log. bin/build bind-mounts $SRC_DIR at /src, so those logs outlive
+      # the container. Without this upload an arch-specific test failure
+      # cannot be diagnosed from CI at all (seen on owe 0.2.7, aarch64).
+      - name: Upload test logs
+        if: always() && steps.build.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-logs-${{ matrix.package }}-${{ matrix.arch }}
+          path: src/**/meson-logs/
+          if-no-files-found: ignore
+          retention-days: 14
       # The artifact label carries the package directory's git tree hash so
       # the publish step can find the build for exactly the tree that merged.
       # The package file inside keeps makepkg's standard name untouched.

.github/workflows/publish.yml

@@ -36,13 +36,18 @@ jobs:
     outputs:
       matrix: ${{ steps.list.outputs.matrix }}
       count: ${{ steps.list.outputs.count }}
+      rebuild: ${{ steps.list.outputs.rebuild }}
+      rebuild_count: ${{ steps.list.outputs.rebuild_count }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           persist-credentials: false
       - id: list
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
+          set -euo pipefail
           if [[ -n "${{ github.event.inputs.packages }}" ]]; then
             names="${{ github.event.inputs.packages }}"
           else
@@ -53,17 +58,102 @@ jobs:
           echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
           echo "count=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT"
           jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix"
+          # Reuse or rebuild, decided per entry and said out loud. An aarch64
+          # tree with no build artifact (PR artifacts last 7 days; a dispatch
+          # may name any package) goes to the rebuild job, which builds it
+          # natively on GitHub's arm64 runner. x86_64 builds inside the
+          # publish job on the droplet, as before.
+          rebuild=()
+          echo "| package | arch | tree | decision |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| --- | --- | --- | --- |" >> "$GITHUB_STEP_SUMMARY"
+          while read -r entry; do
+            package=$(jq -r .package <<<"$entry"); arch=$(jq -r .arch <<<"$entry")
+            hash=$(git rev-parse "HEAD:pkgbuilds/$package")
+            label="$package-$arch-$hash"
+            found=$(curl -fsS -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/repos/${{ github.repository }}/actions/artifacts?name=$label&per_page=5" \
+              | jq -r '[.artifacts[] | select(.expired|not)] | sort_by(.created_at) | last | select(. != null) | "run \(.workflow_run.id), expires \(.expires_at)"')
+            if [[ -n "$found" ]]; then
+              decision="reuse the build artifact ($found)"
+            elif [[ $arch == aarch64 ]]; then
+              decision="no build artifact: rebuild natively on ubuntu-24.04-arm"
+              rebuild+=("$entry")
+            else
+              decision="no build artifact: build in the publish job on the self-hosted builder"
+            fi
+            echo "==> $label: $decision"
+            echo "| $package | $arch | ${hash:0:12} | $decision |" >> "$GITHUB_STEP_SUMMARY"
+          done < <(jq -c '.include[]' <<<"$matrix")
+          echo "rebuild=$(printf '%s\n' "${rebuild[@]}" | jq -sc '{include: .}')" >> "$GITHUB_OUTPUT"
+          echo "rebuild_count=${#rebuild[@]}" >> "$GITHUB_OUTPUT"
+
+  # The aarch64 half of "build it now when there is none". It builds exactly
+  # as build-pr.yml's aarch64 path does (same runner, same builder image,
+  # same bin/build call) and uploads under the same label, so the publish
+  # job collects this run's artifact the way it collects a PR's. No secret
+  # reaches this runner; signing and upload stay on the self-hosted builder.
+  rebuild:
+    needs: changes
+    if: needs.changes.outputs.rebuild_count != '0'
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 180
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.changes.outputs.rebuild) }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      # The same check the publish job makes before building: a re-run for a
+      # package the channel already holds at master's version builds
+      # nothing, and uploads nothing that could shadow the published file.
+      - name: Build ${{ matrix.package }} (${{ matrix.arch }}, native)
+        id: build
+        env:
+          CONTAINER_ENGINE: docker
+        run: |
+          set -euo pipefail
+          plan=$(bin/build --dry-run --mirror edge --arch "${{ matrix.arch }}" --package "${{ matrix.package }}" 2>&1 | grep -E '^==> Plan complete' || true)
+          if [[ -n "$plan" && -z "$(sed -E 's/.*would build: *//' <<<"$plan" | tr -d '[:space:]')" ]]; then
+            echo "==> ${{ matrix.package }} (${{ matrix.arch }}): already published at master's version, nothing to build"
+            echo "built=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          bin/build --mirror edge --arch "${{ matrix.arch }}" --package "${{ matrix.package }}"
+          echo "built=true" >> "$GITHUB_OUTPUT"
+      - name: Pack artifact
+        if: steps.build.outputs.built == 'true'
+        id: pack
+        run: |
+          source helpers/artifact-helpers.sh
+          pack_packages build-output/edge/${{ matrix.arch }} packages.tar
+          tar -tvf packages.tar
+          echo "label=${{ matrix.package }}-${{ matrix.arch }}-$(git rev-parse "HEAD:pkgbuilds/${{ matrix.package }}")" >> "$GITHUB_OUTPUT"
+      - name: Upload artifact
+        if: steps.build.outputs.built == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.pack.outputs.label }}
+          path: packages.tar
+          if-no-files-found: error
+          retention-days: 7
 
   # One job for the whole merge. It collects every PR artifact for the
-  # merged tree (building only what has none), then walks each channel and
+  # merged tree (building only what has none; aarch64 comes from the
+  # rebuild job above), then walks each channel and
   # architecture slot exactly once: pull that database, add every package
   # that belongs in it, upload. Six slots, six round trips, however many
   # packages the merge carried. One process is the only writer, so there
   # is no race between packages; the run-level concurrency group above
   # keeps one merge from overlapping the next.
+  # It waits for the rebuild job and runs whatever that job's result: a
+  # failed rebuild leaves its package without an artifact, and the collect
+  # step below records that and stops before any publish.
   publish:
-    needs: changes
-    if: needs.changes.outputs.count != '0'
+    needs: [changes, rebuild]
+    if: ${{ !cancelled() && needs.changes.result == 'success' && needs.changes.outputs.count != '0' }}
     runs-on: [self-hosted, omarchy-builder]
     environment: publish
     timeout-minutes: 240
@@ -104,15 +194,22 @@ jobs:
             label="$package-$arch-$hash"
             found=$(curl -fsS -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
               "https://api.github.com/repos/${{ github.repository }}/actions/artifacts?name=$label&per_page=5" \
-              | jq -r '[.artifacts[] | select(.expired|not)] | sort_by(.created_at) | last | .archive_download_url // empty')
+              | jq -r '[.artifacts[] | select(.expired|not)] | sort_by(.created_at) | last | select(. != null) | "\(.archive_download_url) \(.workflow_run.id)"')
+            read -r found from_run <<<"$found" || true
             mkdir -p "build-output/edge/$arch"
             if [[ -n "$found" ]]; then
-              echo "==> $label: PR artifact"
+              if [[ $from_run == "${{ github.run_id }}" ]]; then
+                kind=native-rebuild
+                echo "==> $label: artifact from this run's native $arch rebuild"
+              else
+                kind=pr-artifact
+                echo "==> $label: reusing the build artifact from run $from_run"
+              fi
               rm -rf /tmp/artifact; mkdir -p /tmp/artifact
               if curl -fsSL -H "Authorization: Bearer $GH_TOKEN" -o /tmp/artifact.zip "$found" \
                 && unzip -oq /tmp/artifact.zip -d /tmp/artifact \
                 && unpack_packages /tmp/artifact "build-output/edge/$arch"; then
-                jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"pr-artifact"}' >> sources.jsonl
+                jq -nc --arg p "$package" --arg a "$arch" --arg s "$kind" '{package:$p, arch:$a, source:$s}' >> sources.jsonl
               else
                 jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"artifact-download-failed"}' >> sources.jsonl; failed=1; break
               fi
@@ -128,6 +225,14 @@ jobs:
                 jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"already-published"}' >> sources.jsonl
                 continue
               fi
+              # aarch64 never builds here: this droplet is x86 and would
+              # emulate it. No artifact means the native rebuild failed (see
+              # the rebuild job), or an artifact expired between planning
+              # and now (re-run all jobs).
+              if [[ $arch == aarch64 ]]; then
+                echo "::error::$label: no artifact from the native rebuild; not building aarch64 under emulation"
+                jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"native-build-failed"}' >> sources.jsonl; failed=1; break
+              fi
               echo "==> $label: no artifact for this tree, building"
               if OMARCHY_KEEP_BUILD_WORKSPACE=1 bin/build --mirror edge --arch "$arch" --package "$package"; then
                 jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"built"}' >> sources.jsonl
@@ -269,7 +374,7 @@ jobs:
         run: |
           jq -r --arg outcome "${{ needs.publish.result }}" '
             def pkgs: [.packages[] | select(test("-debug-")|not)] | join(", ");
-            def src: if .source=="pr-artifact" then "PR artifact" elif .source=="built" then "built here" elif .source=="already-published" then "already published, skipped" else "**" + .source + "**" end;
+            def src: if .source=="pr-artifact" then "PR artifact" elif .source=="native-rebuild" then "rebuilt natively on arm64" elif .source=="built" then "built here" elif .source=="already-published" then "already published, skipped" else "**" + .source + "**" end;
             "### Publish " + (if $outcome=="success" then "succeeded" else "FAILED" end) +
             " → **" + .target + "**" + (if .target!="live" then " (proof prefix, not live)" else "" end),
             "",
@@ -322,5 +427,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - run: |
-          echo "publish result: ${{ needs.publish.result }}"
+          echo "changes result: ${{ needs.changes.result }}, publish result: ${{ needs.publish.result }}"
+          [[ "${{ needs.changes.result }}" == "success" ]]
           [[ "${{ needs.publish.result }}" == "success" || "${{ needs.publish.result }}" == "skipped" ]]

.github/workflows/sync-upstream.yml

@@ -47,11 +47,13 @@ jobs:
               useradd -m -u "$HOST_UID" -g "$HOST_GID" runner
               chown -R runner:runner /workspace/pkgbuilds
 
+              # The reviewed lane only: packages marked auto_merge ride
+              # track-branches.yml, which merges without a human.
               if [[ -n "${PACKAGES:-}" ]]; then
                 read -r -a package_args <<< "$PACKAGES"
-                runuser -u runner -- ./bin/sync-upstream "${package_args[@]}"
+                runuser -u runner -- ./bin/sync-upstream --lane reviewed "${package_args[@]}"
               else
-                runuser -u runner -- ./bin/sync-upstream
+                runuser -u runner -- ./bin/sync-upstream --lane reviewed
               fi
             '
         env:

.github/workflows/test.yml

@@ -59,6 +59,7 @@ jobs:
               ./tests/partial-release.sh
               ./tests/published-build-plan.sh
               ./tests/settings-boot-config.sh
+              ./tests/pinned-sources.sh
               ./tests/controller.sh
               ./tests/artifact-helpers.sh
               ./tests/limine-mkinitcpio-hook.sh

.github/workflows/track-branches.yml

@@ -0,0 +1,161 @@
+name: Track upstream branches
+
+# The unattended lane. Packages marked "auto_merge": true follow a moving
+# upstream branch (omarchy-dev and omarchy-settings-dev on quattro, omasnap-git
+# on main) rather than tagged releases, so nothing in this repository changes
+# when their source does. This workflow makes each new branch tip a commit pin
+# in the recipe, which publish.yml then treats like any other version bump:
+# the PR builds on the droplets, auto-merge lands it when `result` is green,
+# and the merge publishes the artifacts. A tip that fails to build stays an
+# unmerged red PR that the next tick supersedes.
+#
+# The PR and auto-merge use the PAT in PKGS_BOT_TOKEN so they trigger the
+# build and publish workflows. The built-in GITHUB_TOKEN cannot drive this
+# unattended chain. The PAT needs Contents: write and Pull requests: write
+# on this repository, and its owner must be trusted by the build workflow.
+
+on:
+  schedule:
+    # Every 2 hours, off the hour to dodge the scheduling backlog at :00
+    - cron: '35 */2 * * *'
+  workflow_dispatch:
+    inputs:
+      packages:
+        description: 'Specific packages to track (space-separated, leave empty for every auto_merge package)'
+        required: false
+        default: ''
+
+# One tracker at a time: two runs racing on auto/track-branches would each
+# force-push their own pin over the other's.
+concurrency:
+  group: track-branches
+  cancel-in-progress: false
+
+jobs:
+  track:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Require the tracking token
+        env:
+          PKGS_BOT_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }}
+        run: |
+          if [[ -z "$PKGS_BOT_TOKEN" ]]; then
+            echo "::error::Set PKGS_BOT_TOKEN to a PAT with Contents: write and Pull requests: write on this repository, owned by an account trusted to trigger builds."
+            exit 1
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      # Same container as the reviewed sync: vercmp decides whether a pin is
+      # an upgrade with the comparator pacman uses on users' machines.
+      - name: Pin tracked branches to their current tips
+        id: sync
+        run: |
+          docker run --rm \
+            -e PACKAGES="$PACKAGES" \
+            -e UPSTREAM_GITHUB_TOKEN="$UPSTREAM_GITHUB_TOKEN" \
+            -e HOST_UID="$(id -u)" \
+            -e HOST_GID="$(id -g)" \
+            -v "$PWD/bin:/workspace/bin:ro" \
+            -v "$PWD/helpers:/workspace/helpers:ro" \
+            -v "$PWD/pkgbuilds:/workspace/pkgbuilds" \
+            -w /workspace \
+            archlinux:base-devel bash -lc '
+              set -euo pipefail
+
+              pacman -Syu --noconfirm git jq python libarchive
+
+              groupadd -g "$HOST_GID" runner
+              useradd -m -u "$HOST_UID" -g "$HOST_GID" runner
+              chown -R runner:runner /workspace/pkgbuilds
+
+              if [[ -n "${PACKAGES:-}" ]]; then
+                read -r -a package_args <<< "$PACKAGES"
+                runuser -u runner -- ./bin/sync-upstream --lane auto-merge "${package_args[@]}"
+              else
+                runuser -u runner -- ./bin/sync-upstream --lane auto-merge
+              fi
+            '
+        env:
+          PACKAGES: ${{ github.event.inputs.packages }}
+          UPSTREAM_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check for changes
+        if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }}
+        id: changes
+        run: |
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            git status --porcelain
+            {
+              echo "### Pinned"
+              git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' | grep -E '^\+(pkgver|_commit)=' | sed 's/^+/- /'
+            } >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      # The PR title names what moved, so the merged history reads like a
+      # changelog: "Track upstream branches: omarchy-dev 4.0.0.r6520.g1a2b3c4, ...".
+      - name: Describe the pins
+        if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
+        id: describe
+        run: |
+          title=$(git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' \
+            | awk '/^\+\+\+ b\/pkgbuilds\//{split($2,p,"/"); pkg=p[3]} /^\+pkgver=/{sub(/^\+pkgver=/,""); printf "%s %s, ", pkg, $0}' \
+            | sed 's/, $//')
+          echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT"
+
+      - name: Open or update the tracking PR
+        if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
+        id: pr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.PKGS_BOT_TOKEN }}
+          commit-message: ${{ steps.describe.outputs.title }}
+          title: ${{ steps.describe.outputs.title }}
+          body: |
+            Automated pin of packages that follow a moving upstream branch
+            (`"auto_merge": true` in `.omarchy/package.json`). Each package's
+            `_commit` now points at the branch tip. Fresh tips wait until
+            their commit timestamp is at least `min_release_age` old.
+
+            This PR auto-merges once the build checks pass. A failing build
+            leaves it open; the next tracker run replaces it with the newer tip.
+          branch: auto/track-branches
+          delete-branch: true
+          labels: automated
+
+      # Auto-merge, not a direct merge: branch protection still has to see
+      # `result`, `self-tests` and `build-isolation` green, and this lane
+      # inherits every rule the reviewed lane has except the human.
+      - name: Enable auto-merge
+        if: ${{ !cancelled() && steps.pr.outputs.pull-request-number != '' }}
+        env:
+          GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }}
+          PR: ${{ steps.pr.outputs.pull-request-number }}
+        run: |
+          # Idempotent across re-runs of an updated PR: enabling twice errors.
+          if [[ "$(gh pr view "$PR" -R "${{ github.repository }}" --json autoMergeRequest --jq '.autoMergeRequest != null')" == true ]]; then
+            echo "auto-merge already enabled on #$PR"
+            exit 0
+          fi
+          gh pr merge --auto --merge "$PR" -R "${{ github.repository }}"
+
+      - name: Notify Basecamp on failure
+        if: failure() && env.BASECAMP_CHATBOT_URL != ''
+        env:
+          BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }}
+        run: |
+          curl -s -o /dev/null \
+            -H "Content-Type: application/json" \
+            -d "$(jq -n --arg content \
+              "🔴 <strong>Branch tracking failed</strong><br><a href=\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\">View run</a>" \
+              '{content: $content}')" \
+            "$BASECAMP_CHATBOT_URL"