Merge pull request #511 from omacom/upstream/ci-builds

Build PRs on ephemeral droplets; publish merged packages from CI

.github/VOUCHED.td

@@ -0,0 +1,15 @@
+# Trust list for PR builds.
+#
+# A pull request only builds packages (and spins up builder droplets) when
+# its author is trusted: repository collaborators are trusted automatically
+# and do not need listing; external contributors listed here are trusted
+# too. Anyone else gets the plan only, until a maintainer either adds them
+# here or applies the "build-approved" label to that one PR.
+#
+# Syntax:
+#   github:username
+#   -github:username reason for denouncement
+#
+# Keep entries sorted alphabetically.
+github:f-trycua
+github:scottjones

.github/workflows/build-pr.yml

@@ -0,0 +1,173 @@
+name: Build changed packages
+
+# Build every package directory a PR touches, one job per package per arch, on
+# the self-hosted droplet pool. Artifacts are unsigned; publish.yml signs and
+# publishes them on merge.
+#
+# Tooling runs from the base branch; a PR supplies only pkgbuilds/. The
+# vouch gate limits who may spend compute; this limits what their PR can run.
+
+# No paths filter: `result` is the required status check, so it has to be
+# reported on every PR. A PR that touches no package directory gets an empty
+# matrix and a passing result in seconds.
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+  workflow_dispatch:
+    inputs:
+      packages:
+        description: "Space-separated package directories to build"
+        required: true
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Builds cost real machines, so they run only for trusted authors:
+  # collaborators, anyone in .github/VOUCHED.td (read from the default
+  # branch, so a PR cannot vouch for itself), or a PR a maintainer has
+  # labelled "build-approved". Everyone else gets this job's plan output
+  # and a passing `result`, which is enough for a maintainer to review
+  # before deciding to spend the compute.
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.list.outputs.matrix }}
+      count: ${{ steps.gate.outputs.count }}
+      trusted: ${{ steps.gate.outputs.trusted }}
+    steps:
+      # Same rule as the build job: bin/build-matrix comes from base, the
+      # package directories from the PR head.
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha || github.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - if: github.event_name == 'pull_request'
+        run: |
+          git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.head.sha }}"
+          git checkout "${{ github.event.pull_request.head.sha }}" -- pkgbuilds/
+          # Bootstrap: the PR that introduces this tooling has a base without
+          # it. Take the plan helper from the PR head in that one case; it
+          # runs on a hosted runner and only prints a plan.
+          if [[ ! -x bin/build-matrix ]]; then
+            git checkout "${{ github.event.pull_request.head.sha }}" -- bin/build-matrix helpers/
+            echo "::notice::base branch has no bin/build-matrix; using the PR's copy for planning"
+          fi
+      - id: vouch
+        if: github.event_name == 'pull_request'
+        uses: mitchellh/vouch/action/check-user@f23dbb5e745334f97414ec70463ce7301071a661 # v1
+        with:
+          user: ${{ github.event.pull_request.user.login }}
+          allow-fail: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # One matrix entry per package per architecture. Every package builds
+      # once, against edge; the channels it ships to on merge are carried
+      # along for information. A filename means one set of bytes.
+      - id: list
+        run: |
+          if [[ -n "${{ github.event.inputs.packages }}" ]]; then
+            names="${{ github.event.inputs.packages }}"
+          else
+            names=$(git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" -- pkgbuilds \
+              | awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u)
+          fi
+          matrix=$(printf '%s\n' $names | bin/build-matrix)
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          echo "planned=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT"
+          jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix"
+      - id: gate
+        env:
+          STATUS: ${{ steps.vouch.outputs.status || 'dispatch' }}
+          AUTHOR: ${{ github.event.pull_request.user.login }}
+          APPROVED: ${{ contains(github.event.pull_request.labels.*.name, 'build-approved') }}
+          PLANNED: ${{ steps.list.outputs.planned }}
+        run: |
+          case "$STATUS" in
+            bot|collaborator|vouched|dispatch) trusted=true ;;
+            # A denouncement is absolute: the label cannot override it.
+            denounced) trusted=false ;;
+            *) trusted=$APPROVED ;;
+          esac
+          echo "trusted=$trusted" >> "$GITHUB_OUTPUT"
+          if [[ $trusted == true ]]; then
+            echo "count=$PLANNED" >> "$GITHUB_OUTPUT"
+            echo "Author $AUTHOR is trusted ($STATUS); building $PLANNED package(s)."
+          else
+            echo "count=0" >> "$GITHUB_OUTPUT"
+            echo "::warning::Author $AUTHOR is not trusted ($STATUS). $PLANNED package build(s) planned but not run."
+            if [[ $STATUS == denounced ]]; then
+              echo "::warning::The author is denounced in .github/VOUCHED.td; the build-approved label does not apply."
+            else
+              echo "::warning::A maintainer can add the author to .github/VOUCHED.td, or apply the 'build-approved' label to build this PR."
+            fi
+          fi
+
+  build:
+    needs: changes
+    if: needs.changes.outputs.count != '0'
+    runs-on: [self-hosted, omarchy-builder]
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.changes.outputs.matrix) }}
+    steps:
+      # Tooling from base: everything that executes on this droplet's host
+      # (bin/, helpers/, build/) comes from the base branch. Only the PR's
+      # package directories are overlaid. A PR can therefore change what
+      # gets built, never how the runner builds it. A PR that changes both
+      # tooling and a package builds the package with the OLD tooling; land
+      # the tooling first. workflow_dispatch has no PR and runs as checked out.
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha || github.sha }}
+          persist-credentials: false
+      - name: Overlay the PR's package directories onto base tooling
+        if: github.event_name == 'pull_request'
+        run: |
+          set -euo pipefail
+          git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.head.sha }}"
+          git checkout "${{ github.event.pull_request.head.sha }}" -- pkgbuilds/
+          echo "tooling: $(git rev-parse --short HEAD) (base)  packages: ${{ github.event.pull_request.head.sha }} (PR head)"
+          git status --short | head
+      - name: Build ${{ matrix.package }} (${{ matrix.arch }}, ships to ${{ matrix.channels }})
+        env:
+          CONTAINER_ENGINE: docker
+        run: bin/build --mirror edge --arch ${{ matrix.arch }} --package ${{ matrix.package }}
+      # The artifact label carries the package directory's git tree hash so
+      # the publish step can find the build for exactly the tree that merged.
+      # The package file inside keeps makepkg's standard name untouched.
+      # The artifact label uses the PR head's tree for this package: that is
+      # the tree that merges, and what publish looks up.
+      - name: Tree hash
+        id: tree
+        run: echo "hash=$(git rev-parse "${{ github.event.pull_request.head.sha || github.sha }}:pkgbuilds/${{ matrix.package }}")" >> "$GITHUB_OUTPUT"
+      - name: Upload artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.package }}-${{ matrix.arch }}-${{ steps.tree.outputs.hash }}
+          path: build-output/edge/${{ matrix.arch }}/*.pkg.tar.zst
+          if-no-files-found: error
+          retention-days: 7
+
+  # The one required status check. Matrix job names carry the package name, so
+  # they cannot be listed in branch protection; this job's name is stable and
+  # it fails if any package failed. It also runs (and passes) when no package
+  # changed, so tooling-only PRs are not stuck waiting for a status.
+  result:
+    needs: [changes, build]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "trusted=${{ needs.changes.outputs.trusted }} build=${{ needs.build.result }}"
+          # An untrusted author's PR is held, not failed: the required check
+          # stays pending until a maintainer vouches or labels it.
+          if [[ "${{ needs.changes.outputs.trusted }}" != "true" ]]; then
+            echo "::error::Builds were not run: author is not vouched. Add to .github/VOUCHED.td or apply the 'build-approved' label."
+            exit 1
+          fi
+          [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]]

.github/workflows/publish.yml

@@ -0,0 +1,179 @@
+name: Publish merged packages
+
+# On every push to master: for each package directory the push touched and
+# each architecture it supports, find the PR build artifact for exactly that
+# tree (label = <pkg>-<arch>-<treehash>), or build it now when there is
+# none, then publish that one artifact into every channel the package ships
+# to. One build, one file, several databases: a filename means one set of
+# bytes everywhere, and channels are views over a shared pool.
+#
+# Secrets live in the "publish" environment, restricted to master:
+#   GPG_PRIVATE_KEY, GPG_PASSPHRASE   the channel signing key
+#   R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_ENDPOINT
+# OMARCHY_PUBLISH_PREFIX (environment variable, not secret) points a proof
+# run at a scratch prefix inside the live bucket; empty means the real
+# channel paths.
+
+on:
+  push:
+    branches: [master]
+    paths: ["pkgbuilds/**"]
+  workflow_dispatch:
+    inputs:
+      packages:
+        description: "Space-separated package directories to publish from master"
+        required: true
+
+# Merges serialize. Two publishes into one channel at once would race on
+# the database; queued is fine, cancelled is not.
+concurrency:
+  group: publish
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.list.outputs.matrix }}
+      count: ${{ steps.list.outputs.count }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - id: list
+        run: |
+          if [[ -n "${{ github.event.inputs.packages }}" ]]; then
+            names="${{ github.event.inputs.packages }}"
+          else
+            names=$(git diff --name-only "${{ github.event.before }}..${{ github.sha }}" -- pkgbuilds \
+              | awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u)
+          fi
+          matrix=$(printf '%s\n' $names | bin/build-matrix)
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          echo "count=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT"
+          jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix"
+
+  # One job for the whole merge. It collects every PR artifact for the
+  # merged tree (building only what has none), then walks each channel and
+  # architecture slot exactly once: pull that database, add every package
+  # that belongs in it, upload. Six slots, six round trips, however many
+  # packages the merge carried. One process is the only writer, so there
+  # is no race between packages; the run-level concurrency group above
+  # keeps one merge from overlapping the next.
+  publish:
+    needs: changes
+    if: needs.changes.outputs.count != '0'
+    runs-on: [self-hosted, omarchy-builder]
+    environment: publish
+    timeout-minutes: 240
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      # Every matrix entry, as a file the shell steps can loop over:
+      # package arch channels publish_arches
+      - name: Plan
+        run: |
+          jq -r '.include[] | "\(.package) \(.arch) \(.channels|gsub(" ";",")) \(.publish_arches|gsub(" ";","))"' \
+            <<'EOF_MATRIX' > plan.txt
+          ${{ needs.changes.outputs.matrix }}
+          EOF_MATRIX
+          cat plan.txt
+
+      # Fetch each package's PR artifact into build-output/edge/<arch>/, or
+      # build it when no artifact exists for exactly this tree.
+      - name: Collect artifacts
+        env:
+          GH_TOKEN: ${{ github.token }}
+          CONTAINER_ENGINE: docker
+        run: |
+          set -euo pipefail
+          while read -r package arch channels publish_arches; do
+            hash=$(git rev-parse "HEAD:pkgbuilds/$package")
+            label="$package-$arch-$hash"
+            found=$(curl -fsS -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/repos/${{ github.repository }}/actions/artifacts?name=$label&per_page=5" \
+              | jq -r '[.artifacts[] | select(.expired|not)] | sort_by(.created_at) | last | .archive_download_url // empty')
+            mkdir -p "build-output/edge/$arch"
+            if [[ -n "$found" ]]; then
+              echo "==> $label: PR artifact"
+              curl -fsSL -H "Authorization: Bearer $GH_TOKEN" -o /tmp/artifact.zip "$found"
+              unzip -oq /tmp/artifact.zip -d "build-output/edge/$arch"
+            else
+              echo "==> $label: no artifact for this tree, building"
+              OMARCHY_KEEP_BUILD_WORKSPACE=1 bin/build --mirror edge --arch "$arch" --package "$package"
+            fi
+          done < plan.txt
+          ls -1 build-output/edge/*/*.pkg.tar.zst
+
+      - name: Publish
+        env:
+          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+          RCLONE_CONFIG_R2_TYPE: s3
+          RCLONE_CONFIG_R2_PROVIDER: Cloudflare
+          # The token is scoped to the bucket; it may not CreateBucket, and
+          # rclone's existence check is a CreateBucket in disguise.
+          RCLONE_CONFIG_R2_NO_CHECK_BUCKET: "true"
+          RCLONE_CONFIG_R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          RCLONE_CONFIG_R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+          OMARCHY_PUBLISH_PREFIX: ${{ vars.OMARCHY_PUBLISH_PREFIX }}
+        # repo-add, gpg and bsdtar are Arch tools; run the publish inside the
+        # builder image (host-native, edge) with the workspace mounted.
+        run: |
+          set -euo pipefail
+          docker image inspect omarchy-pkg-builder:latest-x86_64-edge >/dev/null 2>&1 \
+            || docker buildx build --load -t omarchy-pkg-builder:latest-x86_64-edge --build-arg MIRROR=edge build
+
+          # Group the merge's files by the (channel, architecture) slot each
+          # belongs to. A package's files live under build-output/edge/<built
+          # arch>/ and are named <pkgname>-<ver>-<arch|any>.pkg.tar.zst; a
+          # split package's outputs share the pkgbase's directory, so match
+          # on the artifact list rather than the name.
+          # pkgbase is read inside the builder image: the Ubuntu host has no
+          # bsdtar. One container call maps every file to its pkgbase.
+          docker run --rm -v "$PWD:/w:ro" -w /w omarchy-pkg-builder:latest-x86_64-edge bash -c '
+            for f in build-output/edge/*/*.pkg.tar.zst; do
+              printf "%s %s\n" "$f" "$(bsdtar -xOf "$f" .PKGINFO | awk -F" = " "\$1==\"pkgbase\"{print \$2}")"
+            done' > pkgbase.txt
+          declare -A slot_files=()
+          while read -r package arch channels publish_arches; do
+            for f in build-output/edge/"$arch"/*.pkg.tar.zst; do
+              # Only files this package produced (its PKGINFO pkgbase).
+              [[ $(awk -v f="$f" '$1==f{print $2}' pkgbase.txt) == "$package" ]] || continue
+              for mirror in ${channels//,/ }; do
+                for parch in ${publish_arches//,/ }; do
+                  slot_files["$mirror/$parch"]+="$f "
+                done
+              done
+            done
+          done < plan.txt
+
+          # Deterministic slot order: edge before rc before stable, x86_64
+          # before aarch64, so a failure leaves the earlier rings consistent.
+          for mirror in edge rc stable; do
+            for parch in x86_64 aarch64; do
+              files=${slot_files["$mirror/$parch"]:-}
+              [[ -n "$files" ]] || continue
+              echo "==> $mirror/$parch: $files"
+              docker run --rm \
+                -e GPG_PRIVATE_KEY -e GPG_PASSPHRASE -e OMARCHY_PUBLISH_PREFIX \
+                -e RCLONE_CONFIG_R2_TYPE -e RCLONE_CONFIG_R2_PROVIDER -e RCLONE_CONFIG_R2_ENDPOINT -e RCLONE_CONFIG_R2_NO_CHECK_BUCKET \
+                -e RCLONE_CONFIG_R2_ACCESS_KEY_ID -e RCLONE_CONFIG_R2_SECRET_ACCESS_KEY \
+                -v "$PWD:/w:ro" -w /w \
+                omarchy-pkg-builder:latest-x86_64-edge \
+                bin/publish-artifact --remote R2:omarchy-pkgs --mirror "$mirror" --arch "$parch" $files
+            done
+          done
+
+  result:
+    needs: [changes, publish]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "publish result: ${{ needs.publish.result }}"
+          [[ "${{ needs.publish.result }}" == "success" || "${{ needs.publish.result }}" == "skipped" ]]

.github/workflows/test.yml

@@ -1,9 +1,10 @@
 name: Tests
 
+# PR-only. Branch protection requires PRs to be up to date with master, so
+# the PR run already tested the exact tree that merges; a second run on the
+# merge commit would only repeat it. Publishing on push has its own workflow.
 on:
   pull_request:
-  push:
-    branches: [master]
   workflow_dispatch:
 
 jobs:
@@ -45,7 +46,9 @@ jobs:
               ./bin/sync-rebuilds --self-test
               ./bin/omarchy-pkgs self-test
               ./bin/omarchy-release self-test
-              ./tests/dell-xps-touchpad-haptics-install.sh
               ./tests/partial-release.sh
               ./tests/published-build-plan.sh
+              ./tests/controller.sh
+              pacman -S --noconfirm --quiet rclone >/dev/null
+              ./tests/publish-artifact.sh
             '

bin/build

@@ -92,6 +92,8 @@ while [[ $# -gt 0 ]]; do
     echo "  OMARCHY_KEEP_BUILD_WORKSPACE=1   keep build-output and reuse packages already there"
     echo "  OMARCHY_SKIP_BUILDER_IMAGE=1     use the existing builder image instead of building it"
     echo "  OMARCHY_DEFER_RUNTIME_DEPS=true  build the omarchy pair with --nodeps (transaction verified later)"
+    echo "  OMARCHY_PUBLISHED_REPO_URL=<url> channel to plan and resolve against when no local tree exists"
+    echo "                                   (default https://pkgs.omarchy.org; empty disables the fallback)"
     echo ""
     exit 0
     ;;
@@ -256,6 +258,7 @@ DOCKER_ARGS=(
   -e MIRROR="$MIRROR"
   -e PACKAGES="$PACKAGES"
   -e OMARCHY_RC_PINS="${OMARCHY_RC_PINS:-}"
+  -e OMARCHY_PUBLISHED_REPO_URL="${OMARCHY_PUBLISHED_REPO_URL-https://pkgs.omarchy.org}"
   -e DEFER_RUNTIME_DEPS="$DEFER_RUNTIME_DEPS"
   -e BUILD_PLAN_DIR=/build-plan
   -v "$PLAN_DIR:/build-plan"

bin/build-matrix

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Print the PR build matrix for a set of package directories as JSON: one
+# entry per package per supported architecture. Every package builds exactly
+# once, against edge, and that one artifact is what every channel ships:
+# channels are databases over a shared pool of files, and a filename must
+# mean one set of bytes. "channels" lists where the artifact is published on
+# merge: edge for everything, plus rc and stable immediately for the fast
+# ring. Eligibility comes from package_builds_for_mirror, the rule the
+# release host uses, so CI and the host cannot disagree.
+#
+# Usage: build-matrix [--arch <arch>|all] <package>...
+#   Reads package names on stdin when none are given. With no --arch, every
+#   architecture in CI_ARCHES (default "x86_64 aarch64") the package supports.
+#   Output: {"include":[{"package":"x","arch":"x86_64","channels":"edge rc stable","publish_arches":"x86_64"},...]}
+#   arch is where it builds; publish_arches lists every architecture
+#   database the file goes into (all of them for arch=any).
+set -euo pipefail
+BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
+source "$BUILD_ROOT/helpers/paths.sh"
+source "$BUILD_ROOT/helpers/package-metadata.sh"
+
+ARCHES=${CI_ARCHES:-x86_64 aarch64}
+if [[ "${1:-}" == --arch ]]; then [[ $2 == all ]] || ARCHES=$2; shift 2; fi
+for a in $ARCHES; do require_valid_arch "$a"; done
+
+if (( $# )); then names=("$@"); else mapfile -t names; fi
+
+entries=()
+for name in "${names[@]}"; do
+  [[ -n "$name" ]] || continue
+  pkgdir="$PKGBUILDS_DIR/$name"
+  [[ -d "$pkgdir" ]] || continue
+  # skip_build packages still build on their own PR (explicit --package
+  # semantics); the host's unscoped runs are what skip them.
+  channels=""
+  for mirror in $VALID_MIRRORS; do
+    package_builds_for_mirror "$pkgdir" "$mirror" && channels="$channels $mirror"
+  done
+  channels=${channels# }
+  [[ -n "$channels" ]] || continue
+  # An arch=any package produces one architecture-independent file, so it
+  # builds once, on the first architecture, and that file serves every
+  # channel database of every architecture.
+  if [[ " $(package_arches "$pkgdir" "${ARCHES%% *}") " == *" any "* ]]; then
+    entries+=("$(jq -nc --arg p "$name" --arg a "${ARCHES%% *}" --arg c "$channels" --arg pa "$ARCHES" '{package:$p, arch:$a, channels:$c, publish_arches:$pa}')")
+    continue
+  fi
+  for arch in $ARCHES; do
+    package_supports_arch "$pkgdir" "$arch" || continue
+    entries+=("$(jq -nc --arg p "$name" --arg a "$arch" --arg c "$channels" '{package:$p, arch:$a, channels:$c, publish_arches:$a}')")
+  done
+done
+
+printf '%s\n' "${entries[@]}" | jq -sc '{include: .}'

bin/publish-artifact

@@ -0,0 +1,118 @@
+#!/bin/bash
+# Publish built packages into one channel of the remote repository,
+# incrementally and immutably.
+#
+#   publish-artifact --mirror <edge|rc|stable> --arch <arch> <pkg files...>
+#
+# What it does, in order:
+#   1. pull the channel's current database from the remote
+#   2. refuse if any package filename already exists on the remote
+#   3. sign each package (GPG_PRIVATE_KEY / GPG_PASSPHRASE)
+#   4. repo-add the packages into the pulled database (replaces the entry
+#      for that name; nothing else in the channel is touched)
+#   5. upload packages, then signatures, then the database last
+#
+# Never overwrites: uploads use --ignore-existing for packages and the
+# pre-check in step 2 makes a same-name collision a hard failure rather than
+# a silent skip. The database is the only object rewritten, and it is
+# uploaded only after every file it references is present.
+#
+# The remote is an rclone remote (REMOTE, default the production one);
+# OMARCHY_PUBLISH_PREFIX can point a proof run at a scratch prefix.
+set -euo pipefail
+BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
+source "$BUILD_ROOT/helpers/message-helpers.sh"
+source "$BUILD_ROOT/helpers/paths.sh"
+
+REMOTE=${REMOTE:-pkgs.omarchy.org:omarchy-pkgs}
+PREFIX=${OMARCHY_PUBLISH_PREFIX:-}
+FILES=()
+while [[ $# -gt 0 ]]; do
+  case $1 in
+  --mirror) MIRROR=$2; require_valid_mirror "$MIRROR"; shift 2 ;;
+  --arch) ARCH=$2; require_valid_arch "$ARCH"; shift 2 ;;
+  --remote) REMOTE=$2; shift 2 ;;
+  -h|--help) sed -n '2,22p' "$0"; exit 0 ;;
+  -*) print_error "Unknown option: $1"; exit 1 ;;
+  *) FILES+=("$1"); shift ;;
+  esac
+done
+(( ${#FILES[@]} )) || { print_error "No package files given"; exit 1; }
+: "${GPG_PRIVATE_KEY:?}"; GPG_PASSPHRASE=${GPG_PASSPHRASE-}
+
+DEST="$REMOTE/${PREFIX:+$PREFIX/}$MIRROR/$ARCH"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+print_header "Publish to $DEST"
+
+# --- 0. sanity: every file is a package, named as makepkg names it ---------
+for f in "${FILES[@]}"; do
+  [[ -f "$f" && "$f" == *.pkg.tar.* && "$f" != *.sig ]] || { print_error "Not a package file: $f"; exit 1; }
+  name=$(bsdtar -xOf "$f" .PKGINFO | awk -F' = ' '$1=="pkgname"{print $2}')
+  ver=$(bsdtar -xOf "$f" .PKGINFO | awk -F' = ' '$1=="pkgver"{print $2}')
+  pkgarch=$(bsdtar -xOf "$f" .PKGINFO | awk -F' = ' '$1=="arch"{print $2}')
+  [[ "$(basename "$f")" == "$name-$ver-$pkgarch.pkg.tar."* ]] || {
+    print_error "Filename does not match PKGINFO ($name-$ver-$pkgarch): $(basename "$f")"; exit 1; }
+  [[ "$pkgarch" == any || "$pkgarch" == "$ARCH" ]] || { print_error "$f is $pkgarch, publishing to $ARCH"; exit 1; }
+done
+
+# --- 1. pull the current database -----------------------------------------
+mkdir -p "$WORK/repo"
+listing=$(rclone lsf "$DEST/" --s3-no-head 2>/dev/null || true)
+if grep -q '^omarchy.db.tar.zst$' <<<"$listing"; then
+  rclone copy "$DEST/omarchy.db.tar.zst" "$WORK/repo" --s3-no-head
+  rclone copy "$DEST/omarchy.files.tar.zst" "$WORK/repo" --s3-no-head 2>/dev/null || true
+  print_info "Pulled current database ($(tar -tf "$WORK/repo/omarchy.db.tar.zst" | grep -c '/$') entries)"
+else
+  print_warning "No database at $DEST — creating a new one"
+fi
+
+# --- 2. same-name collisions ----------------------------------------------
+# A filename must mean one set of bytes across every channel. The same file
+# reaching a channel that already holds it (a fast-ring publish after edge,
+# a re-run, a later promotion) is fine: it is skipped on upload and only the
+# database entry is added. Different bytes under a name the channel already
+# has is the one thing this must never do.
+for f in "${FILES[@]}"; do
+  b=$(basename "$f")
+  grep -qxF "$b" <<<"$listing" || continue
+  remote_sum=$(rclone hashsum md5 "$DEST/$b" --s3-no-head 2>/dev/null | awk '{print $1}')
+  local_sum=$(md5sum "$f" | awk '{print $1}')
+  if [[ -n "$remote_sum" && "$remote_sum" == "$local_sum" ]]; then
+    print_info "Already published with identical bytes, adding to the database only: $b"
+  else
+    print_error "Already published with DIFFERENT bytes, refusing to overwrite: $b"
+    echo "  Bump pkgrel; published filenames are immutable."
+    exit 1
+  fi
+done
+
+# --- 3. sign ---------------------------------------------------------------
+export GNUPGHOME="$WORK/gnupg"; mkdir -m 700 "$GNUPGHOME"
+echo "$GPG_PRIVATE_KEY" | gpg --batch --quiet --import
+KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '$1=="sec"{print $5; exit}')
+[[ -n "$KEY_ID" ]] || { print_error "No secret key imported"; exit 1; }
+for f in "${FILES[@]}"; do
+  cp "$f" "$WORK/repo/"
+  gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" \
+    --detach-sign --no-armor --local-user "$KEY_ID" "$WORK/repo/$(basename "$f")"
+  print_step "signed $(basename "$f")"
+done
+
+# --- 4. repo-add (replaces the entry for each pkgname) ---------------------
+( cd "$WORK/repo" && repo-add --quiet omarchy.db.tar.zst "${FILES[@]/#*\//}" )
+ln -sf omarchy.db.tar.zst "$WORK/repo/omarchy.db"
+ln -sf omarchy.files.tar.zst "$WORK/repo/omarchy.files"
+print_info "Database now has $(tar -tf "$WORK/repo/omarchy.db.tar.zst" | grep -c '/$') entries"
+
+# --- 5. upload: packages, signatures, database last -----------------------
+rclone copy "$WORK/repo" "$DEST" --s3-no-head --ignore-existing --filter '- *.sig' --filter '+ *.pkg.tar.*' --filter '- *'
+rclone copy "$WORK/repo" "$DEST" --s3-no-head --ignore-existing --filter '+ *.pkg.tar.*.sig' --filter '- *'
+# Re-verify every referenced file is really there before the db goes up.
+listing=$(rclone lsf "$DEST/" --s3-no-head)
+for f in "${FILES[@]}"; do
+  b=$(basename "$f")
+  grep -qxF "$b" <<<"$listing" && grep -qxF "$b.sig" <<<"$listing" || { print_error "Upload incomplete: $b"; exit 1; }
+done
+rclone copy "$WORK/repo" "$DEST" --s3-no-head --checksum --copy-links --filter '+ omarchy.db*' --filter '+ omarchy.files*' --filter '- *'
+print_success "Published ${#FILES[@]} package(s) to $DEST"

build/Dockerfile

@@ -129,6 +129,7 @@ RUN pacman -Syu --noconfirm && \
         wget \
         curl \
         jq \
+        rclone \
         gnupg && \
     pacman -Scc --noconfirm && \
     rm -rf /var/cache/pacman/pkg/*
@@ -146,7 +147,8 @@ RUN useradd -m -G wheel -s /bin/bash builder && \
 # be skipped at signing. Pin the extension so both architectures match.
 RUN sed -i 's/^#MAKEFLAGS=.*/MAKEFLAGS="-j$(nproc)"/' /etc/makepkg.conf && \
     sed -i 's/^COMPRESSZST=.*/COMPRESSZST=(zstd -c -z -q --threads=0 -)/' /etc/makepkg.conf && \
-    sed -i "s|^PKGEXT=.*|PKGEXT='.pkg.tar.zst'|" /etc/makepkg.conf
+    sed -i "s|^PKGEXT=.*|PKGEXT='.pkg.tar.zst'|" /etc/makepkg.conf && \
+    sed -i 's|^#\?PACKAGER=.*|PACKAGER="Omarchy <pkgs@omarchy.org>"|' /etc/makepkg.conf
 
 # Pacman wrapper that auto-resolves package conflicts (e.g. rustup vs rust).
 # makepkg only passes --noconfirm to pacman, which defaults to 'N' on conflict

build/build.sh

@@ -26,6 +26,29 @@ DEFER_RUNTIME_DEPS=${DEFER_RUNTIME_DEPS:-false}
 
 source "$HELPERS_DIR/package-metadata.sh"
 
+# Where the channel's published database is read from for planning. On the
+# repository host it is the published tree itself. Anywhere else (a CI runner,
+# a fresh clone) that tree is absent, so the database is fetched from the
+# public channel and the same URL serves as pacman's dependency repository.
+# Set OMARCHY_PUBLISHED_REPO_URL= (empty) to disable the remote fallback.
+PUBLISHED_REPO_URL=${OMARCHY_PUBLISHED_REPO_URL-https://pkgs.omarchy.org}
+PUBLISHED_DB_DIR="$FINAL_OUTPUT_DIR"
+PUBLISHED_REPO_SERVER=""
+if [[ ! -f "$FINAL_OUTPUT_DIR/omarchy.db.tar.zst" && ! -f "$FINAL_OUTPUT_DIR/omarchy.db" && -n "$PUBLISHED_REPO_URL" ]]; then
+  remote_channel="$PUBLISHED_REPO_URL/$MIRROR/$ARCH"
+  remote_db_dir=$(mktemp -d /tmp/omarchy-published.XXXXXX) || exit 1
+  # Cache-bust: the channel sits behind a CDN that serves a stale database
+  # for a while after a sync.
+  if curl -fsSL "$remote_channel/omarchy.db.tar.zst?$(date +%s)" -o "$remote_db_dir/omarchy.db.tar.zst"; then
+    PUBLISHED_DB_DIR="$remote_db_dir"
+    PUBLISHED_REPO_SERVER="$remote_channel"
+    echo "==> No local published tree; planning against $remote_channel"
+  else
+    rm -rf "$remote_db_dir"
+    echo "==> No local published tree and $remote_channel is unavailable; treating the channel as empty"
+  fi
+fi
+
 if [[ $DEFER_RUNTIME_DEPS != "false" && $DEFER_RUNTIME_DEPS != "true" ]]; then
   echo "DEFER_RUNTIME_DEPS must be true or false" >&2
   exit 1
@@ -118,10 +141,15 @@ if [[ "$DRY_RUN" != true ]]; then
   fi
   touch "$BUILD_PLAN_DIR/repository-initialized" || exit 1
 
-  # Add omarchy repo if it has a database (stable packages)
+  # Add omarchy repo if it has a database (stable packages). The local tree
+  # is trusted as-is; the public channel is verified against the omarchy
+  # keyring the image already carries.
   if [[ -f "$FINAL_OUTPUT_DIR/omarchy.db.tar.zst" ]] || [[ -f "$FINAL_OUTPUT_DIR/omarchy.db" ]]; then
     sudo sed -i "/^\[core\]$/i [omarchy]\nSigLevel = Optional TrustAll\nServer = file://$FINAL_OUTPUT_DIR\n" /etc/pacman.conf
     echo "  -> omarchy (priority 2): $FINAL_OUTPUT_DIR"
+  elif [[ -n "$PUBLISHED_REPO_SERVER" ]]; then
+    sudo sed -i "/^\[core\]$/i [omarchy]\nSigLevel = Required DatabaseOptional\nServer = $PUBLISHED_REPO_SERVER\n" /etc/pacman.conf
+    echo "  -> omarchy (priority 2): $PUBLISHED_REPO_SERVER"
   fi
 
   # Sync pacman database
@@ -159,10 +187,10 @@ LOCAL_VERSION_CACHE_LOADED=false
 LOCAL_VERSION_CACHE_DB=""
 
 load_local_versions() {
-  local db="$FINAL_OUTPUT_DIR/omarchy.db.tar.zst"
+  local db="$PUBLISHED_DB_DIR/omarchy.db.tar.zst"
 
   if [[ ! -f "$db" ]]; then
-    db="$FINAL_OUTPUT_DIR/omarchy.db"
+    db="$PUBLISHED_DB_DIR/omarchy.db"
   fi
 
   [[ -f "$db" ]] || return 0

ci/README.md

@@ -0,0 +1,79 @@
+# CI spike: build PRs on ephemeral DigitalOcean droplets
+
+Status: spike. Nothing here publishes. The repository host keeps building and
+signing on merge exactly as before.
+
+## Pieces
+
+- `.github/workflows/build-pr.yml` — on a PR touching `pkgbuilds/**`, one job
+  per changed package on runners labelled `omarchy-builder`. Uploads the
+  unsigned `.pkg.tar.zst` as a workflow artifact (7 days).
+- `runner-cloud-init.yaml` — Ubuntu 24.04 user-data: docker + buildx, the
+  GitHub runner registered `--ephemeral`, runs one job, powers off.
+- `controller.sh` — systemd timer every minute on a small always-on droplet.
+  Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet (ric1) per job up
+  to `MAX_DROPLETS`, deletes droplets that are powered off or older than
+  `MAX_AGE_MINUTES`. No inbound endpoint. Plain curl against both APIs, no
+  doctl and no gh: a token in the environment cannot pick the wrong account
+  the way a saved doctl context can. Needs curl and jq.
+  `tests/controller.sh` exercises every decision against canned responses.
+- `controller-box/` — the always-on droplet: unit, timer, env template,
+  cloud-init, and `create.sh` to stand it up with one API call.
+
+## Standing up the controller box
+
+    DIGITALOCEAN_TOKEN=<omarchy account> GITHUB_TOKEN=<fine-grained PAT> \
+      REPO=omacom/omarchy-pkgs ci/controller-box/create.sh <branch>
+
+The GitHub PAT is fine-grained, scoped to the one repo: Actions read,
+Administration read+write (registration tokens). The DO token is baked into
+the box's env file, so it is the account that pays for builder droplets.
+Watch it with `journalctl -u omarchy-controller -f` on the box.
+
+## What the spike proved (2026-09-17, fork ryanrhughes/omarchy-pkgs)
+
+- `bin/build` works from a bare clone: with no local published tree it
+  plans against and resolves from `https://pkgs.omarchy.org/<mirror>/<arch>`.
+- Droplet create → runner registered: ~70 s. omarchy-fish PR job: 2 min
+  including the builder image build. Droplet powers off after the job.
+- linux-omarchy on a c-32 droplet: 30 min wall clock for the build job
+  (23:39 → 00:09), 254 MB artifact. Cold start ~90 s before the job began.
+- A PR whose PKGBUILD fails to build turns the required check red and GitHub
+  refuses the merge (`mergeStateStatus=BLOCKED`, `gh pr merge` refuses
+  without `--admin`).
+- Controller: one queued job + one busy droplet ⇒ creates exactly one more;
+  reaps powered-off droplets on the next tick.
+
+## Not done (required before this touches the real repo)
+
+- Tooling from base: check out master's `bin/ helpers/ build/` and overlay
+  only the PR's `pkgbuilds/<name>`; today a PR can edit the build script
+  and it runs on the droplet. The vouch gate limits who can do that, not
+  what they can do.
+- DigitalOcean cloud firewall on the `omarchy-builder` tag: no inbound, no
+  egress to private ranges or the metadata address.
+- A fine-grained GitHub token for the real repository (the one on the
+  controller box is scoped to the fork), and the publish environment's
+  secrets set there.
+- Disable the host's auto-release timers for any channel CI publishes to,
+  so two writers never touch one database.
+
+## Done since the spike README was first written
+
+- Controller as a systemd timer on its own droplet, plain curl, self-test.
+- Build once against edge; one artifact per package per architecture,
+  published into every channel it belongs to (fast ring: all three at
+  once). arch=any builds once for every architecture database.
+- Publish is incremental and immutable: pull the channel db, refuse
+  different bytes under an existing name, accept identical bytes, upload
+  packages then signatures then the db.
+- aarch64 under QEMU with credential-preserving binfmt.
+- Vouch gate: collaborators, `.github/VOUCHED.td`, or the `build-approved`
+  label; denounced authors cannot be overridden by the label.
+- Tests run on PRs only; `result`, `self-tests`, `build-isolation` are the
+  required checks with strict up-to-date branches.
+
+## Cleanup
+
+    doctl compute droplet list --tag-name omarchy-builder
+    doctl compute droplet delete -f <id>

ci/controller-box/cloud-init.yaml

@@ -0,0 +1,45 @@
+#cloud-config
+# The always-on controller droplet (smallest size is fine). Clones the repo
+# for ci/controller.sh, installs the unit and timer, and starts polling.
+#
+# Substitute before use:
+#   __REPO_URL__   https://github.com/omacom/omarchy-pkgs.git
+#   __BRANCH__     branch carrying ci/ (master once merged)
+#   __ENV_B64__    base64 of a filled-in controller.env.example
+#   __SSH_KEYS_JSON__ JSON array of public keys authorized for root
+package_update: true
+packages: [curl, jq, git]
+
+# Root stays reachable by key so the journal can be read. Two things stand
+# in the way on DO images: disable_root rewrites root's keys into a stub, and
+# with no account ssh key attached DO expires root's password, which makes
+# sshd refuse every non-interactive session with "password change required".
+disable_root: false
+chpasswd:
+  expire: false
+ssh_authorized_keys: __SSH_KEYS_JSON__
+
+users:
+  - name: controller
+    shell: /bin/bash
+
+write_files:
+  # defer: write after the users module has created the controller group,
+  # otherwise chown to root:controller fails and the unit cannot read this.
+  - path: /etc/omarchy-controller.env
+    permissions: "0640"
+    owner: root:controller
+    encoding: b64
+    defer: true
+    content: __ENV_B64__
+
+runcmd:
+  - chage -d "$(date +%F)" -M -1 root
+  - chown root:controller /etc/omarchy-controller.env && chmod 0640 /etc/omarchy-controller.env
+  - git clone --depth 1 --branch __BRANCH__ __REPO_URL__ /opt/omarchy-pkgs
+  - mkdir -p /run/omarchy-controller && chown controller:controller /run/omarchy-controller
+  - echo "d /run/omarchy-controller 0755 controller controller -" > /etc/tmpfiles.d/omarchy-controller.conf
+  # runcmd is executed by /bin/sh: no brace expansion.
+  - cp /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.service /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.timer /etc/systemd/system/
+  - systemctl daemon-reload
+  - systemctl enable --now omarchy-controller.timer

ci/controller-box/controller.env.example

@@ -0,0 +1,15 @@
+# /etc/omarchy-controller.env — mode 0600, owned by root, read by systemd.
+DIGITALOCEAN_TOKEN=dop_v1_...
+# Fine-grained PAT scoped to the repo: Actions: read, Administration: read+write
+GITHUB_TOKEN=github_pat_...
+REPO=omacom/omarchy-pkgs
+LABEL=omarchy-builder
+TAG=omarchy-builder
+REGION=ric1
+SIZE=g5-32vcpu-64gb-50gb
+MAX_DROPLETS=6
+MAX_AGE_MINUTES=200
+LOCK=/run/omarchy-controller/lock
+# Operator public keys for root on every builder droplet (JSON array).
+# create.sh fills this from the operators' GitHub keys.
+SSH_KEYS_JSON=[]

ci/controller-box/create.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Create the controller droplet with plain curl. Run from a laptop, once.
+#
+#   DIGITALOCEAN_TOKEN=... GITHUB_TOKEN=... ci/controller-box/create.sh [branch]
+#
+# The DO token given here is baked into the box's env file, so it must be the
+# token for the account that should pay for builder droplets.
+set -euo pipefail
+here=$(dirname "$0")
+: "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}"
+REPO=${REPO:-omacom/omarchy-pkgs}
+BRANCH=${1:-master}
+REGION=${REGION:-ric1}
+NAME=${NAME:-omarchy-controller}
+# Optional DO ssh key ids as a JSON array, e.g. SSH_KEYS='[123]', for reading
+# the journal while bringing the box up. Not needed once it works.
+SSH_KEYS=${SSH_KEYS:-[]}
+# Public keys authorized for root: the operators' GitHub keys, fetched at
+# creation so the box never depends on an ssh_key API scope. Override with
+# ADMIN_GITHUB_USERS.
+ADMIN_GITHUB_USERS=${ADMIN_GITHUB_USERS:-ryanrhughes dhh}
+ssh_keys_json=$(for u in $ADMIN_GITHUB_USERS; do curl -fsS "https://github.com/$u.keys"; done | jq -R . | jq -sc .)
+[[ $(jq length <<<"$ssh_keys_json") -gt 0 ]] || { echo "no ssh keys fetched for $ADMIN_GITHUB_USERS" >&2; exit 1; }
+
+env_file=$(sed -e "s|^DIGITALOCEAN_TOKEN=.*|DIGITALOCEAN_TOKEN=$DIGITALOCEAN_TOKEN|" \
+               -e "s|^GITHUB_TOKEN=.*|GITHUB_TOKEN=$GITHUB_TOKEN|" \
+               -e "s|^REPO=.*|REPO=$REPO|" \
+               -e "s|^SSH_KEYS_JSON=.*|SSH_KEYS_JSON=$ssh_keys_json|" "$here/controller.env.example")
+userdata=$(sed -e "s|__REPO_URL__|https://github.com/$REPO.git|" -e "s|__BRANCH__|$BRANCH|" \
+               -e "s|__ENV_B64__|$(printf '%s\n' "$env_file" | base64 -w0)|" \
+               -e "s|__SSH_KEYS_JSON__|$ssh_keys_json|" "$here/cloud-init.yaml")
+body=$(jq -n --arg name "$NAME" --arg region "$REGION" --arg ud "$userdata" --argjson keys "$SSH_KEYS" \
+  '{name:$name, region:$region, size:"s-1vcpu-1gb", image:"ubuntu-24-04-x64", tags:["omarchy-controller"], user_data:$ud, ssh_keys:$keys}')
+
+# Refuse to create a second one.
+existing=$(curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
+  "https://api.digitalocean.com/v2/droplets?tag_name=omarchy-controller" | jq '.droplets | length')
+if (( existing > 0 )); then echo "a controller droplet already exists" >&2; exit 1; fi
+
+curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" -H "Content-Type: application/json" \
+  -X POST -d "$body" https://api.digitalocean.com/v2/droplets | jq -r '"created \(.droplet.name) id=\(.droplet.id)"'

ci/controller-box/omarchy-controller.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Provision ephemeral omarchy-builder runner droplets for queued jobs
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=controller
+EnvironmentFile=/etc/omarchy-controller.env
+ExecStart=/opt/omarchy-pkgs/ci/controller.sh
+# The reaper's safety net is time, not state; a hung tick must not hold the lock.
+TimeoutStartSec=240

ci/controller-box/omarchy-controller.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run the omarchy-builder controller every minute
+
+[Timer]
+OnBootSec=1min
+OnUnitActiveSec=1min
+AccuracySec=5s
+
+[Install]
+WantedBy=timers.target

ci/controller.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+# Droplet-per-job controller for the omarchy-builder runner pool.
+#
+# Run from a systemd timer every minute on a small always-on droplet. No
+# inbound endpoint: it polls GitHub for queued jobs wanting our label, creates
+# one ephemeral droplet per job (up to MAX_DROPLETS), and deletes droplets
+# that have powered off or exceeded MAX_AGE_MINUTES. The reaper does not
+# trust its own bookkeeping: it lists by tag and acts on what DigitalOcean
+# reports.
+#
+# Talks to both APIs with curl. No doctl: its saved contexts silently choose
+# an account; a token in the environment cannot. Needs curl and jq.
+#
+# Environment:
+#   DIGITALOCEAN_TOKEN   DO API token for the account that pays for droplets
+#   GITHUB_TOKEN         fine-grained PAT: Actions read, Administration write
+#   REPO                 owner/name
+set -euo pipefail
+
+REPO=${REPO:?owner/name}
+: "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}"
+LABEL=${LABEL:-omarchy-builder}
+TAG=${TAG:-omarchy-builder}
+REGION=${REGION:-ric1}
+SIZE=${SIZE:-g5-32vcpu-64gb-50gb}
+IMAGE=${IMAGE:-ubuntu-24-04-x64}
+MAX_DROPLETS=${MAX_DROPLETS:-4}
+MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200}
+RUNNER_VERSION=${RUNNER_VERSION:-2.337.0}
+CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml}
+# Operator public keys authorized on every builder (JSON array of strings).
+# The box's env file carries them; empty means no root login.
+SSH_KEYS_JSON=${SSH_KEYS_JSON:-[]}
+LOCK=${LOCK:-/tmp/omarchy-controller.lock}
+
+log() { echo "$(date '+%F %T') $*"; }
+
+# The only two places the outside world is touched. The self-test overrides
+# both, so every decision below is exercised against canned responses.
+do_api() { # do_api <path> [curl args...]
+  local path=$1; shift
+  curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
+    -H "Content-Type: application/json" "https://api.digitalocean.com/v2/$path" "$@"
+}
+gh_api() { # gh_api <path> [curl args...]
+  local path=$1; shift
+  curl -fsS -H "Authorization: Bearer $GITHUB_TOKEN" \
+    -H "Accept: application/vnd.github+json" "https://api.github.com/$path" "$@"
+}
+
+# --- reap ------------------------------------------------------------------
+reap() {
+  local now id status created age
+  now=$(date +%s)
+  while read -r id status created; do
+    [[ -n "$id" ]] || continue
+    age=$(( (now - $(date -d "$created" +%s)) / 60 ))
+    if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )); then
+      log "deleting droplet $id (status=$status age=${age}m)"
+      do_api "droplets/$id" -X DELETE
+    fi
+  done < <(do_api "droplets?tag_name=$TAG&per_page=200" |
+    jq -r '.droplets[] | "\(.id) \(.status) \(.created_at)"')
+}
+
+# --- demand ----------------------------------------------------------------
+queued_jobs() {
+  local run
+  gh_api "repos/$REPO/actions/runs?status=queued&per_page=50" --get \
+    | jq -r '.workflow_runs[].id' |
+  while read -r run; do
+    gh_api "repos/$REPO/actions/runs/$run/jobs" \
+      | jq -r --arg l "$LABEL" '.jobs[] | select(.status=="queued") | select(.labels | index($l)) | .id'
+  done | wc -l
+}
+
+live_droplets() {
+  do_api "droplets?tag_name=$TAG&per_page=200" | jq '[.droplets[] | select(.status != "off")] | length'
+}
+
+busy_runners() {
+  gh_api "repos/$REPO/actions/runners?per_page=100" \
+    | jq --arg l "$LABEL" '[.runners[] | select(.busy) | select(any(.labels[]; .name == $l))] | length'
+}
+
+# --- create ----------------------------------------------------------------
+create_droplet() {
+  local token userdata name body
+  token=$(gh_api "repos/$REPO/actions/runners/registration-token" -X POST | jq -r .token)
+  userdata=$(sed -e "s|__REPO__|$REPO|g" -e "s|__RUNNER_TOKEN__|$token|g" \
+                 -e "s|__RUNNER_LABELS__|$LABEL|g" -e "s|__RUNNER_VERSION__|$RUNNER_VERSION|g" \
+                 -e "s|__SSH_KEYS_JSON__|$SSH_KEYS_JSON|" "$CLOUD_INIT")
+  name="$TAG-$(date +%s)-$RANDOM"
+  body=$(jq -n --arg name "$name" --arg region "$REGION" --arg size "$SIZE" --arg image "$IMAGE" \
+    --arg tag "$TAG" --arg ud "$userdata" \
+    '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}')
+  log "creating $name ($SIZE)"
+  do_api droplets -X POST -d "$body" | jq -r '"created droplet \(.droplet.id)"'
+}
+
+controller_tick() {
+  reap
+  local queued live busy available need room
+  queued=$(queued_jobs)
+  live=$(live_droplets)
+  busy=$(busy_runners)
+  # A live droplet whose runner is busy is spoken for. Only droplets still
+  # booting or listening can absorb a queued job.
+  available=$(( live - busy )); (( available < 0 )) && available=0
+  need=$(( queued - available ))
+  (( need > 0 )) || return 0
+  room=$(( MAX_DROPLETS - live ))
+  (( need > room )) && need=$room
+  if (( need <= 0 )); then
+    log "at cap ($live/$MAX_DROPLETS, $busy busy) with $queued queued"
+    return 0
+  fi
+  local i
+  for (( i = 0; i < need; i++ )); do create_droplet; done
+}
+
+if [[ "${CONTROLLER_LIBRARY_ONLY:-}" != 1 ]]; then
+  exec 9>"$LOCK"; flock -n 9 || exit 0
+  controller_tick
+fi

ci/runner-cloud-init.yaml

@@ -0,0 +1,77 @@
+#cloud-config
+# Ephemeral GitHub Actions runner for omarchy-pkgs package builds.
+#
+# Boots an Ubuntu droplet, installs docker and the runner, registers ONCE with
+# --ephemeral, runs exactly one job, then powers off. The controller (or the
+# reaper) deletes the powered-off droplet. Nothing here holds a long-lived
+# credential: the registration token is single-use and expires in an hour.
+#
+# Substitute before use:
+#   __REPO__            owner/name
+#   __RUNNER_TOKEN__    registration token (gh api -X POST repos/O/R/actions/runners/registration-token)
+#   __RUNNER_LABELS__   e.g. omarchy-builder
+#   __RUNNER_VERSION__  e.g. 2.329.0
+
+# Operators can reach a builder by key while it lives; it powers off after
+# one job anyway. Keys are substituted by the controller (__SSH_KEYS_JSON__).
+disable_root: false
+chpasswd:
+  expire: false
+ssh_authorized_keys: __SSH_KEYS_JSON__
+
+package_update: true
+packages:
+  - docker.io
+  - docker-buildx
+  - unzip
+  - git
+  - curl
+  - jq
+  - rsync
+
+users:
+  - name: runner
+    groups: [docker]
+    shell: /bin/bash
+    sudo: ALL=(ALL) NOPASSWD:ALL
+
+write_files:
+  # defer: write after users/groups exist, so /home/runner is created by
+  # useradd (owned by runner) rather than by this module as root.
+  - path: /home/runner/start.sh
+    permissions: "0755"
+    owner: runner:runner
+    defer: true
+    content: |
+      #!/bin/bash
+      set -euo pipefail
+      cd /home/runner
+      mkdir -p actions-runner && cd actions-runner
+      arch=$(uname -m); [[ $arch == x86_64 ]] && arch=x64
+      curl -fsSL -o runner.tgz \
+        "https://github.com/actions/runner/releases/download/v__RUNNER_VERSION__/actions-runner-linux-${arch}-__RUNNER_VERSION__.tar.gz"
+      tar xzf runner.tgz && rm runner.tgz
+      ./config.sh --unattended --ephemeral \
+        --url "https://github.com/__REPO__" \
+        --token "__RUNNER_TOKEN__" \
+        --name "do-$(hostname)" \
+        --labels "__RUNNER_LABELS__" \
+        --replace
+      ./run.sh
+      # One job done. Power off; the controller deletes powered-off droplets.
+      sudo poweroff
+
+runcmd:
+  # With no account ssh key attached, DO expires root's password, and sshd
+  # then refuses every non-interactive session. Clear it first so operators
+  # can read the logs of a builder that never registers.
+  - chage -d "$(date +%F)" -M -1 root
+  - systemctl enable --now docker
+  # aarch64 builds run under user-mode emulation (DO has no arm droplets).
+  # Register QEMU with the F and C flags via the multiarch image, exactly as
+  # helpers/docker-helpers.sh setup_qemu does: Ubuntu's qemu-user-static
+  # package registers without C, so sudo inside the emulated container fails
+  # with "effective uid is not 0". Best-effort: an x86-only job never needs it.
+  - docker run --rm --privileged docker.io/multiarch/qemu-user-static --reset -p yes --credential yes || true
+  - chown -R runner:runner /home/runner
+  - sudo -u runner /home/runner/start.sh > /home/runner/runner.log 2>&1

tests/controller.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Self-test for ci/controller.sh: every decision, no cloud.
+#
+# The controller's two API functions are overridden with canned responses and
+# a recorder, then each scenario asserts which creates and deletes it issued.
+set -euo pipefail
+ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
+
+export REPO=o/r DIGITALOCEAN_TOKEN=x GITHUB_TOKEN=x
+export CLOUD_INIT="$ROOT/ci/runner-cloud-init.yaml" LOCK=/tmp/controller-test.lock
+CONTROLLER_LIBRARY_ONLY=1 source "$ROOT/ci/controller.sh"
+
+# Calls are recorded to a file: the controller invokes the API functions
+# inside command substitutions, and a subshell cannot append to an array.
+CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT
+NOW=$(date -u +%FT%TZ)
+OLD=$(date -u -d '5 hours ago' +%FT%TZ)
+
+# Scenario state: DROPLETS is "id status created" lines, QUEUED a count,
+# BUSY a count.
+do_api() {
+  local path=$1; shift
+  echo "do $path $*" >>"$CALLS_FILE"
+  case "$path" in
+    droplets\?*) printf '%s\n' "$DROPLETS" | jq -Rs '{droplets: [split("\n")[] | select(length>0) | split(" ") | {id: .[0]|tonumber, status: .[1], created_at: .[2]}]}' ;;
+    droplets) echo '{"droplet":{"id":999}}' ;;
+    droplets/*) echo '{}' ;;
+  esac
+}
+gh_api() {
+  local path=$1; shift
+  echo "gh $path $*" >>"$CALLS_FILE"
+  case "$path" in
+    */actions/runs\?*) jq -nc --argjson n "$QUEUED" '{workflow_runs: [range($n) | {id: .}]}' ;;
+    */actions/runs/*/jobs) echo '{"jobs":[{"id":1,"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;;
+    */actions/runners\?*) jq -nc --argjson n "$BUSY" '{runners: [range($n) | {busy: true, labels: [{name: "omarchy-builder"}]}]}' ;;
+    */registration-token) echo '{"token":"T"}' ;;
+  esac
+}
+
+creates() { grep -c '^do droplets -X POST' "$CALLS_FILE" || true; }
+deletes() { grep -c '^do droplets/.* -X DELETE' "$CALLS_FILE" || true; }
+run() { : >"$CALLS_FILE"; controller_tick >/dev/null; }
+check() { # check <name> <expected creates> <expected deletes>
+  local c d; c=$(creates); d=$(deletes)
+  if [[ "$c" == "$2" && "$d" == "$3" ]]; then echo "PASS: $1"; else echo "FAIL: $1 (creates=$c want $2, deletes=$d want $3)"; cat "$CALLS_FILE"; exit 1; fi
+}
+
+DROPLETS="" QUEUED=0 BUSY=0; run; check "idle: nothing queued, nothing to reap" 0 0
+DROPLETS="" QUEUED=2 BUSY=0; run; check "two queued, none live: create two" 2 0
+DROPLETS="1 active $NOW" QUEUED=1 BUSY=1; run; check "one queued, one live but busy: create one" 1 0
+DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0
+DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1
+DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1
+DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0
+DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0
+DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1
+
+# The create body must carry the tag (reaper scope) and substituted user-data.
+BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT
+do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; }
+gh_api() { echo '{"token":"TOK"}'; }
+create_droplet >/dev/null
+jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.user_data | test("--token \"TOK\"")) and (.user_data | test("__") | not)' "$BODY_FILE" >/dev/null \
+  && echo "PASS: create body carries tag, size, substituted user-data" \
+  || { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; }

tests/publish-artifact.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Self-test for bin/publish-artifact against a local directory as the remote.
+# Needs repo-add, gpg, rclone, bsdtar (run in the Arch builder/test container).
+set -euo pipefail
+ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
+T=$(mktemp -d); chmod 755 "$T"; trap 'rm -rf "$T"' EXIT
+REMOTE="$T/r2"; mkdir -p "$REMOTE"
+
+# throwaway signing key
+export GNUPGHOME="$T/g"; mkdir -m700 "$GNUPGHOME"
+gpg --batch --quiet --passphrase '' --quick-gen-key 'Test <t@t>' ed25519 sign 0 2>/dev/null
+export GPG_PRIVATE_KEY=$(gpg --batch --armor --export-secret-keys 'Test <t@t>') GPG_PASSPHRASE=''
+unset GNUPGHOME
+
+# minimal real packages via makepkg
+mkpkg() { # mkpkg <name> <pkgrel> <arch> [payload]
+  local d="$T/src/$1-$2${4:+-$4}"; mkdir -p "$d"; cd "$d"
+  printf 'pkgname=%s\npkgver=1.0\npkgrel=%s\narch=(%s)\npackage(){ install -Dm644 /dev/null "$pkgdir/usr/share/%s-%s"; echo "%s" > "$pkgdir/usr/share/%s-%s"; }\n' "$1" "$2" "$3" "$1" "$2" "${4:-payload}" "$1" "$2" > PKGBUILD
+  # CARCH so the PKGINFO records the requested arch (--ignorearch would
+  # stamp the host's).
+  # makepkg refuses to run as root (the CI test container does); build the
+  # fixture as an unprivileged user in that case.
+  if (( EUID == 0 )); then
+    id -u fixture >/dev/null 2>&1 || useradd -m fixture
+    chmod 755 "$T/src"; chown -R fixture "$d"
+    runuser -u fixture -- env CARCH=$3 makepkg -f --nodeps --ignorearch >/dev/null 2>&1
+  else
+    CARCH=$3 makepkg -f --nodeps --ignorearch >/dev/null 2>&1
+  fi
+  ls "$d"/*.pkg.tar.zst
+}
+A1=$(mkpkg alpha 1 any); A2=$(mkpkg alpha 2 any); B1=$(mkpkg beta 1 x86_64); C1=$(mkpkg gamma 1 aarch64)
+
+pub() { "$ROOT/bin/publish-artifact" --remote "$REMOTE" --mirror edge --arch x86_64 "$@" >"$T/out" 2>&1; }
+entries() { tar -tf "$REMOTE/edge/x86_64/omarchy.db.tar.zst" | grep '/$' | sort | tr '\n' ' '; }
+pass() { echo "PASS: $1"; }
+fail() { echo "FAIL: $1"; cat "$T/out"; exit 1; }
+
+pub "$A1" && [[ "$(entries)" == "alpha-1.0-1/ " ]] && [[ -f "$REMOTE/edge/x86_64/$(basename "$A1").sig" ]] \
+  && pass "first publish creates db with one entry and a signature" || fail "first publish"
+
+sum_before=$(sha256sum "$REMOTE/edge/x86_64/$(basename "$A1")")
+pub "$B1" && [[ "$(entries)" == "alpha-1.0-1/ beta-1.0-1/ " ]] && [[ "$(sha256sum "$REMOTE/edge/x86_64/$(basename "$A1")")" == "$sum_before" ]] \
+  && pass "second package added incrementally; first file untouched" || fail "incremental add"
+
+pub "$A2" && [[ "$(entries)" == "alpha-1.0-2/ beta-1.0-1/ " ]] && [[ -f "$REMOTE/edge/x86_64/$(basename "$A1")" ]] \
+  && pass "new pkgrel replaces the db entry, old file remains on remote" || fail "replace entry"
+
+# Same bytes again: allowed, idempotent (this is how a fast-ring artifact
+# reaches rc and stable after edge, and how a re-run recovers).
+pub "$A2" && grep -q 'identical bytes' "$T/out" && [[ "$(entries)" == "alpha-1.0-2/ beta-1.0-1/ " ]] \
+  && pass "identical bytes under an existing name: accepted, db unchanged" || fail "identical republish"
+
+# Orphan repair: a file that reached the remote but whose db entry was lost
+# (a concurrent publish overwrote the db) is fixed by publishing it again.
+( cd "$REMOTE/edge/x86_64" && repo-remove --quiet omarchy.db.tar.zst alpha >/dev/null 2>&1 )
+[[ "$(entries)" == "beta-1.0-1/ " ]] || fail "fixture: could not drop alpha from the db"
+pub "$A2" && [[ "$(entries)" == "alpha-1.0-2/ beta-1.0-1/ " ]] \
+  && pass "orphaned file regains its db entry on republish" || fail "orphan repair"
+
+# Different bytes under an existing name: refused. Build alpha-2 again with
+# a different payload (makepkg is reproducible, so the content must change).
+A2b=$(mkpkg alpha 2 any different-payload)
+[[ "$(md5sum < "$A2")" != "$(md5sum < "$A2b")" ]] || { echo "fixture: rebuilt package is byte-identical, cannot test"; exit 1; }
+if pub "$A2b"; then fail "different bytes under same filename should refuse"; else grep -q 'DIFFERENT bytes' "$T/out" && pass "different bytes under an existing name refused" || fail "wrong refusal reason"; fi
+
+if pub "$C1"; then fail "aarch64 package into x86_64 should refuse"; else grep -q 'publishing to x86_64' "$T/out" && pass "wrong-arch package refused" || fail "wrong-arch reason"; fi
+
+cp "$B1" "$T/renamed-1.0-1-x86_64.pkg.tar.zst"
+if pub "$T/renamed-1.0-1-x86_64.pkg.tar.zst"; then fail "filename/PKGINFO mismatch should refuse"; else grep -q 'does not match PKGINFO' "$T/out" && pass "filename must match PKGINFO" || fail "mismatch reason"; fi
+
+# db must verify: pacman can read it and each package's signature checks
+gpg --batch --quiet --import <<<"$GPG_PRIVATE_KEY" 2>/dev/null || true
+( cd "$REMOTE/edge/x86_64" && for f in *.pkg.tar.zst; do gpg --batch --quiet --verify "$f.sig" "$f" 2>/dev/null || { echo "FAIL: signature $f"; exit 1; }; done ) && pass "all signatures verify"