diff --git a/.github/VOUCHED.td b/.github/VOUCHED.td new file mode 100644 index 0000000..2aec3ae --- /dev/null +++ b/.github/VOUCHED.td @@ -0,0 +1,15 @@ +# Trust list for PR builds. +# +# A pull request only builds packages (and spins up builder droplets) when +# its author is trusted: repository collaborators are trusted automatically +# and do not need listing; external contributors listed here are trusted +# too. Anyone else gets the plan only, until a maintainer either adds them +# here or applies the "build-approved" label to that one PR. +# +# Syntax: +# github:username +# -github:username reason for denouncement +# +# Keep entries sorted alphabetically. +github:f-trycua +github:scottjones diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 0000000..8d68dbb --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,166 @@ +name: Build changed packages + +# Build every package directory a PR touches, one job per package per arch, on +# the self-hosted droplet pool. Artifacts are unsigned; publish.yml signs and +# publishes them on merge. +# +# Tooling runs from the base branch; a PR supplies only pkgbuilds/. The +# vouch gate limits who may spend compute; this limits what their PR can run. + +# No paths filter: `result` is the required status check, so it has to be +# reported on every PR. A PR that touches no package directory gets an empty +# matrix and a passing result in seconds. +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + packages: + description: "Space-separated package directories to build" + required: true + +concurrency: + group: build-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + # Builds cost real machines, so they run only for trusted authors: + # collaborators, anyone in .github/VOUCHED.td (read from the default + # branch, so a PR cannot vouch for itself), or a PR a maintainer has + # labelled "build-approved". Everyone else gets this job's plan output + # and a passing `result`, which is enough for a maintainer to review + # before deciding to spend the compute. + changes: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.list.outputs.matrix }} + count: ${{ steps.gate.outputs.count }} + trusted: ${{ steps.gate.outputs.trusted }} + steps: + # Same rule as the build job: bin/build-matrix comes from base, the + # package directories from the PR head. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - if: github.event_name == 'pull_request' + run: | + git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.head.sha }}" + git checkout "${{ github.event.pull_request.head.sha }}" -- pkgbuilds/ + - id: vouch + if: github.event_name == 'pull_request' + uses: mitchellh/vouch/action/check-user@f23dbb5e745334f97414ec70463ce7301071a661 # v1 + with: + user: ${{ github.event.pull_request.user.login }} + allow-fail: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # One matrix entry per package per architecture. Every package builds + # once, against edge; the channels it ships to on merge are carried + # along for information. A filename means one set of bytes. + - id: list + run: | + if [[ -n "${{ github.event.inputs.packages }}" ]]; then + names="${{ github.event.inputs.packages }}" + else + names=$(git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" -- pkgbuilds \ + | awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u) + fi + matrix=$(printf '%s\n' $names | bin/build-matrix) + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "planned=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT" + jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix" + - id: gate + env: + STATUS: ${{ steps.vouch.outputs.status || 'dispatch' }} + AUTHOR: ${{ github.event.pull_request.user.login }} + APPROVED: ${{ contains(github.event.pull_request.labels.*.name, 'build-approved') }} + PLANNED: ${{ steps.list.outputs.planned }} + run: | + case "$STATUS" in + bot|collaborator|vouched|dispatch) trusted=true ;; + # A denouncement is absolute: the label cannot override it. + denounced) trusted=false ;; + *) trusted=$APPROVED ;; + esac + echo "trusted=$trusted" >> "$GITHUB_OUTPUT" + if [[ $trusted == true ]]; then + echo "count=$PLANNED" >> "$GITHUB_OUTPUT" + echo "Author $AUTHOR is trusted ($STATUS); building $PLANNED package(s)." + else + echo "count=0" >> "$GITHUB_OUTPUT" + echo "::warning::Author $AUTHOR is not trusted ($STATUS). $PLANNED package build(s) planned but not run." + if [[ $STATUS == denounced ]]; then + echo "::warning::The author is denounced in .github/VOUCHED.td; the build-approved label does not apply." + else + echo "::warning::A maintainer can add the author to .github/VOUCHED.td, or apply the 'build-approved' label to build this PR." + fi + fi + + build: + needs: changes + if: needs.changes.outputs.count != '0' + runs-on: [self-hosted, omarchy-builder] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.changes.outputs.matrix) }} + steps: + # Tooling from base: everything that executes on this droplet's host + # (bin/, helpers/, build/) comes from the base branch. Only the PR's + # package directories are overlaid. A PR can therefore change what + # gets built, never how the runner builds it. A PR that changes both + # tooling and a package builds the package with the OLD tooling; land + # the tooling first. workflow_dispatch has no PR and runs as checked out. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false + - name: Overlay the PR's package directories onto base tooling + if: github.event_name == 'pull_request' + run: | + set -euo pipefail + git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.head.sha }}" + git checkout "${{ github.event.pull_request.head.sha }}" -- pkgbuilds/ + echo "tooling: $(git rev-parse --short HEAD) (base) packages: ${{ github.event.pull_request.head.sha }} (PR head)" + git status --short | head + - name: Build ${{ matrix.package }} (${{ matrix.arch }}, ships to ${{ matrix.channels }}) + env: + CONTAINER_ENGINE: docker + run: bin/build --mirror edge --arch ${{ matrix.arch }} --package ${{ matrix.package }} + # The artifact label carries the package directory's git tree hash so + # the publish step can find the build for exactly the tree that merged. + # The package file inside keeps makepkg's standard name untouched. + # The artifact label uses the PR head's tree for this package: that is + # the tree that merges, and what publish looks up. + - name: Tree hash + id: tree + run: echo "hash=$(git rev-parse "${{ github.event.pull_request.head.sha || github.sha }}:pkgbuilds/${{ matrix.package }}")" >> "$GITHUB_OUTPUT" + - name: Upload artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.package }}-${{ matrix.arch }}-${{ steps.tree.outputs.hash }} + path: build-output/edge/${{ matrix.arch }}/*.pkg.tar.zst + if-no-files-found: error + retention-days: 7 + + # The one required status check. Matrix job names carry the package name, so + # they cannot be listed in branch protection; this job's name is stable and + # it fails if any package failed. It also runs (and passes) when no package + # changed, so tooling-only PRs are not stuck waiting for a status. + result: + needs: [changes, build] + if: always() + runs-on: ubuntu-latest + steps: + - run: | + echo "trusted=${{ needs.changes.outputs.trusted }} build=${{ needs.build.result }}" + # An untrusted author's PR is held, not failed: the required check + # stays pending until a maintainer vouches or labels it. + if [[ "${{ needs.changes.outputs.trusted }}" != "true" ]]; then + echo "::error::Builds were not run: author is not vouched. Add to .github/VOUCHED.td or apply the 'build-approved' label." + exit 1 + fi + [[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]] diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..9eb969c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,179 @@ +name: Publish merged packages + +# On every push to master: for each package directory the push touched and +# each architecture it supports, find the PR build artifact for exactly that +# tree (label = --), or build it now when there is +# none, then publish that one artifact into every channel the package ships +# to. One build, one file, several databases: a filename means one set of +# bytes everywhere, and channels are views over a shared pool. +# +# Secrets live in the "publish" environment, restricted to master: +# GPG_PRIVATE_KEY, GPG_PASSPHRASE the channel signing key +# R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_ENDPOINT +# OMARCHY_PUBLISH_PREFIX (environment variable, not secret) points a proof +# run at a scratch prefix inside the live bucket; empty means the real +# channel paths. + +on: + push: + branches: [master] + paths: ["pkgbuilds/**"] + workflow_dispatch: + inputs: + packages: + description: "Space-separated package directories to publish from master" + required: true + +# Merges serialize. Two publishes into one channel at once would race on +# the database; queued is fine, cancelled is not. +concurrency: + group: publish + cancel-in-progress: false + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.list.outputs.matrix }} + count: ${{ steps.list.outputs.count }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - id: list + run: | + if [[ -n "${{ github.event.inputs.packages }}" ]]; then + names="${{ github.event.inputs.packages }}" + else + names=$(git diff --name-only "${{ github.event.before }}..${{ github.sha }}" -- pkgbuilds \ + | awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u) + fi + matrix=$(printf '%s\n' $names | bin/build-matrix) + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "count=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT" + jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix" + + # One job for the whole merge. It collects every PR artifact for the + # merged tree (building only what has none), then walks each channel and + # architecture slot exactly once: pull that database, add every package + # that belongs in it, upload. Six slots, six round trips, however many + # packages the merge carried. One process is the only writer, so there + # is no race between packages; the run-level concurrency group above + # keeps one merge from overlapping the next. + publish: + needs: changes + if: needs.changes.outputs.count != '0' + runs-on: [self-hosted, omarchy-builder] + environment: publish + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + # Every matrix entry, as a file the shell steps can loop over: + # package arch channels publish_arches + - name: Plan + run: | + jq -r '.include[] | "\(.package) \(.arch) \(.channels|gsub(" ";",")) \(.publish_arches|gsub(" ";","))"' \ + <<'EOF_MATRIX' > plan.txt + ${{ needs.changes.outputs.matrix }} + EOF_MATRIX + cat plan.txt + + # Fetch each package's PR artifact into build-output/edge//, or + # build it when no artifact exists for exactly this tree. + - name: Collect artifacts + env: + GH_TOKEN: ${{ github.token }} + CONTAINER_ENGINE: docker + run: | + set -euo pipefail + while read -r package arch channels publish_arches; do + hash=$(git rev-parse "HEAD:pkgbuilds/$package") + label="$package-$arch-$hash" + found=$(curl -fsS -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/actions/artifacts?name=$label&per_page=5" \ + | jq -r '[.artifacts[] | select(.expired|not)] | sort_by(.created_at) | last | .archive_download_url // empty') + mkdir -p "build-output/edge/$arch" + if [[ -n "$found" ]]; then + echo "==> $label: PR artifact" + curl -fsSL -H "Authorization: Bearer $GH_TOKEN" -o /tmp/artifact.zip "$found" + unzip -oq /tmp/artifact.zip -d "build-output/edge/$arch" + else + echo "==> $label: no artifact for this tree, building" + OMARCHY_KEEP_BUILD_WORKSPACE=1 bin/build --mirror edge --arch "$arch" --package "$package" + fi + done < plan.txt + ls -1 build-output/edge/*/*.pkg.tar.zst + + - name: Publish + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + RCLONE_CONFIG_R2_TYPE: s3 + RCLONE_CONFIG_R2_PROVIDER: Cloudflare + # The token is scoped to the bucket; it may not CreateBucket, and + # rclone's existence check is a CreateBucket in disguise. + RCLONE_CONFIG_R2_NO_CHECK_BUCKET: "true" + RCLONE_CONFIG_R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + RCLONE_CONFIG_R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + OMARCHY_PUBLISH_PREFIX: ${{ vars.OMARCHY_PUBLISH_PREFIX }} + # repo-add, gpg and bsdtar are Arch tools; run the publish inside the + # builder image (host-native, edge) with the workspace mounted. + run: | + set -euo pipefail + docker image inspect omarchy-pkg-builder:latest-x86_64-edge >/dev/null 2>&1 \ + || docker buildx build --load -t omarchy-pkg-builder:latest-x86_64-edge --build-arg MIRROR=edge build + + # Group the merge's files by the (channel, architecture) slot each + # belongs to. A package's files live under build-output/edge// and are named --.pkg.tar.zst; a + # split package's outputs share the pkgbase's directory, so match + # on the artifact list rather than the name. + # pkgbase is read inside the builder image: the Ubuntu host has no + # bsdtar. One container call maps every file to its pkgbase. + docker run --rm -v "$PWD:/w:ro" -w /w omarchy-pkg-builder:latest-x86_64-edge bash -c ' + for f in build-output/edge/*/*.pkg.tar.zst; do + printf "%s %s\n" "$f" "$(bsdtar -xOf "$f" .PKGINFO | awk -F" = " "\$1==\"pkgbase\"{print \$2}")" + done' > pkgbase.txt + declare -A slot_files=() + while read -r package arch channels publish_arches; do + for f in build-output/edge/"$arch"/*.pkg.tar.zst; do + # Only files this package produced (its PKGINFO pkgbase). + [[ $(awk -v f="$f" '$1==f{print $2}' pkgbase.txt) == "$package" ]] || continue + for mirror in ${channels//,/ }; do + for parch in ${publish_arches//,/ }; do + slot_files["$mirror/$parch"]+="$f " + done + done + done + done < plan.txt + + # Deterministic slot order: edge before rc before stable, x86_64 + # before aarch64, so a failure leaves the earlier rings consistent. + for mirror in edge rc stable; do + for parch in x86_64 aarch64; do + files=${slot_files["$mirror/$parch"]:-} + [[ -n "$files" ]] || continue + echo "==> $mirror/$parch: $files" + docker run --rm \ + -e GPG_PRIVATE_KEY -e GPG_PASSPHRASE -e OMARCHY_PUBLISH_PREFIX \ + -e RCLONE_CONFIG_R2_TYPE -e RCLONE_CONFIG_R2_PROVIDER -e RCLONE_CONFIG_R2_ENDPOINT -e RCLONE_CONFIG_R2_NO_CHECK_BUCKET \ + -e RCLONE_CONFIG_R2_ACCESS_KEY_ID -e RCLONE_CONFIG_R2_SECRET_ACCESS_KEY \ + -v "$PWD:/w:ro" -w /w \ + omarchy-pkg-builder:latest-x86_64-edge \ + bin/publish-artifact --remote R2:omarchy-pkgs --mirror "$mirror" --arch "$parch" $files + done + done + + result: + needs: [changes, publish] + if: always() + runs-on: ubuntu-latest + steps: + - run: | + echo "publish result: ${{ needs.publish.result }}" + [[ "${{ needs.publish.result }}" == "success" || "${{ needs.publish.result }}" == "skipped" ]] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ac36ec..3ff7ea7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,10 @@ name: Tests +# PR-only. Branch protection requires PRs to be up to date with master, so +# the PR run already tested the exact tree that merges; a second run on the +# merge commit would only repeat it. Publishing on push has its own workflow. on: pull_request: - push: - branches: [master] workflow_dispatch: jobs: @@ -45,7 +46,9 @@ jobs: ./bin/sync-rebuilds --self-test ./bin/omarchy-pkgs self-test ./bin/omarchy-release self-test - ./tests/dell-xps-touchpad-haptics-install.sh ./tests/partial-release.sh ./tests/published-build-plan.sh + ./tests/controller.sh + pacman -S --noconfirm --quiet rclone >/dev/null + ./tests/publish-artifact.sh ' diff --git a/bin/build b/bin/build index 728852b..765e071 100755 --- a/bin/build +++ b/bin/build @@ -92,6 +92,8 @@ while [[ $# -gt 0 ]]; do echo " OMARCHY_KEEP_BUILD_WORKSPACE=1 keep build-output and reuse packages already there" echo " OMARCHY_SKIP_BUILDER_IMAGE=1 use the existing builder image instead of building it" echo " OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy pair with --nodeps (transaction verified later)" + echo " OMARCHY_PUBLISHED_REPO_URL= channel to plan and resolve against when no local tree exists" + echo " (default https://pkgs.omarchy.org; empty disables the fallback)" echo "" exit 0 ;; @@ -256,6 +258,7 @@ DOCKER_ARGS=( -e MIRROR="$MIRROR" -e PACKAGES="$PACKAGES" -e OMARCHY_RC_PINS="${OMARCHY_RC_PINS:-}" + -e OMARCHY_PUBLISHED_REPO_URL="${OMARCHY_PUBLISHED_REPO_URL-https://pkgs.omarchy.org}" -e DEFER_RUNTIME_DEPS="$DEFER_RUNTIME_DEPS" -e BUILD_PLAN_DIR=/build-plan -v "$PLAN_DIR:/build-plan" diff --git a/bin/build-matrix b/bin/build-matrix new file mode 100755 index 0000000..e772349 --- /dev/null +++ b/bin/build-matrix @@ -0,0 +1,54 @@ +#!/bin/bash +# Print the PR build matrix for a set of package directories as JSON: one +# entry per package per supported architecture. Every package builds exactly +# once, against edge, and that one artifact is what every channel ships: +# channels are databases over a shared pool of files, and a filename must +# mean one set of bytes. "channels" lists where the artifact is published on +# merge: edge for everything, plus rc and stable immediately for the fast +# ring. Eligibility comes from package_builds_for_mirror, the rule the +# release host uses, so CI and the host cannot disagree. +# +# Usage: build-matrix [--arch |all] ... +# Reads package names on stdin when none are given. With no --arch, every +# architecture in CI_ARCHES (default "x86_64 aarch64") the package supports. +# Output: {"include":[{"package":"x","arch":"x86_64","channels":"edge rc stable","publish_arches":"x86_64"},...]} +# arch is where it builds; publish_arches lists every architecture +# database the file goes into (all of them for arch=any). +set -euo pipefail +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/package-metadata.sh" + +ARCHES=${CI_ARCHES:-x86_64 aarch64} +if [[ "${1:-}" == --arch ]]; then [[ $2 == all ]] || ARCHES=$2; shift 2; fi +for a in $ARCHES; do require_valid_arch "$a"; done + +if (( $# )); then names=("$@"); else mapfile -t names; fi + +entries=() +for name in "${names[@]}"; do + [[ -n "$name" ]] || continue + pkgdir="$PKGBUILDS_DIR/$name" + [[ -d "$pkgdir" ]] || continue + # skip_build packages still build on their own PR (explicit --package + # semantics); the host's unscoped runs are what skip them. + channels="" + for mirror in $VALID_MIRRORS; do + package_builds_for_mirror "$pkgdir" "$mirror" && channels="$channels $mirror" + done + channels=${channels# } + [[ -n "$channels" ]] || continue + # An arch=any package produces one architecture-independent file, so it + # builds once, on the first architecture, and that file serves every + # channel database of every architecture. + if [[ " $(package_arches "$pkgdir" "${ARCHES%% *}") " == *" any "* ]]; then + entries+=("$(jq -nc --arg p "$name" --arg a "${ARCHES%% *}" --arg c "$channels" --arg pa "$ARCHES" '{package:$p, arch:$a, channels:$c, publish_arches:$pa}')") + continue + fi + for arch in $ARCHES; do + package_supports_arch "$pkgdir" "$arch" || continue + entries+=("$(jq -nc --arg p "$name" --arg a "$arch" --arg c "$channels" '{package:$p, arch:$a, channels:$c, publish_arches:$a}')") + done +done + +printf '%s\n' "${entries[@]}" | jq -sc '{include: .}' diff --git a/bin/publish-artifact b/bin/publish-artifact new file mode 100755 index 0000000..32a4909 --- /dev/null +++ b/bin/publish-artifact @@ -0,0 +1,118 @@ +#!/bin/bash +# Publish built packages into one channel of the remote repository, +# incrementally and immutably. +# +# publish-artifact --mirror --arch +# +# What it does, in order: +# 1. pull the channel's current database from the remote +# 2. refuse if any package filename already exists on the remote +# 3. sign each package (GPG_PRIVATE_KEY / GPG_PASSPHRASE) +# 4. repo-add the packages into the pulled database (replaces the entry +# for that name; nothing else in the channel is touched) +# 5. upload packages, then signatures, then the database last +# +# Never overwrites: uploads use --ignore-existing for packages and the +# pre-check in step 2 makes a same-name collision a hard failure rather than +# a silent skip. The database is the only object rewritten, and it is +# uploaded only after every file it references is present. +# +# The remote is an rclone remote (REMOTE, default the production one); +# OMARCHY_PUBLISH_PREFIX can point a proof run at a scratch prefix. +set -euo pipefail +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +source "$BUILD_ROOT/helpers/message-helpers.sh" +source "$BUILD_ROOT/helpers/paths.sh" + +REMOTE=${REMOTE:-pkgs.omarchy.org:omarchy-pkgs} +PREFIX=${OMARCHY_PUBLISH_PREFIX:-} +FILES=() +while [[ $# -gt 0 ]]; do + case $1 in + --mirror) MIRROR=$2; require_valid_mirror "$MIRROR"; shift 2 ;; + --arch) ARCH=$2; require_valid_arch "$ARCH"; shift 2 ;; + --remote) REMOTE=$2; shift 2 ;; + -h|--help) sed -n '2,22p' "$0"; exit 0 ;; + -*) print_error "Unknown option: $1"; exit 1 ;; + *) FILES+=("$1"); shift ;; + esac +done +(( ${#FILES[@]} )) || { print_error "No package files given"; exit 1; } +: "${GPG_PRIVATE_KEY:?}"; GPG_PASSPHRASE=${GPG_PASSPHRASE-} + +DEST="$REMOTE/${PREFIX:+$PREFIX/}$MIRROR/$ARCH" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +print_header "Publish to $DEST" + +# --- 0. sanity: every file is a package, named as makepkg names it --------- +for f in "${FILES[@]}"; do + [[ -f "$f" && "$f" == *.pkg.tar.* && "$f" != *.sig ]] || { print_error "Not a package file: $f"; exit 1; } + name=$(bsdtar -xOf "$f" .PKGINFO | awk -F' = ' '$1=="pkgname"{print $2}') + ver=$(bsdtar -xOf "$f" .PKGINFO | awk -F' = ' '$1=="pkgver"{print $2}') + pkgarch=$(bsdtar -xOf "$f" .PKGINFO | awk -F' = ' '$1=="arch"{print $2}') + [[ "$(basename "$f")" == "$name-$ver-$pkgarch.pkg.tar."* ]] || { + print_error "Filename does not match PKGINFO ($name-$ver-$pkgarch): $(basename "$f")"; exit 1; } + [[ "$pkgarch" == any || "$pkgarch" == "$ARCH" ]] || { print_error "$f is $pkgarch, publishing to $ARCH"; exit 1; } +done + +# --- 1. pull the current database ----------------------------------------- +mkdir -p "$WORK/repo" +listing=$(rclone lsf "$DEST/" --s3-no-head 2>/dev/null || true) +if grep -q '^omarchy.db.tar.zst$' <<<"$listing"; then + rclone copy "$DEST/omarchy.db.tar.zst" "$WORK/repo" --s3-no-head + rclone copy "$DEST/omarchy.files.tar.zst" "$WORK/repo" --s3-no-head 2>/dev/null || true + print_info "Pulled current database ($(tar -tf "$WORK/repo/omarchy.db.tar.zst" | grep -c '/$') entries)" +else + print_warning "No database at $DEST — creating a new one" +fi + +# --- 2. same-name collisions ---------------------------------------------- +# A filename must mean one set of bytes across every channel. The same file +# reaching a channel that already holds it (a fast-ring publish after edge, +# a re-run, a later promotion) is fine: it is skipped on upload and only the +# database entry is added. Different bytes under a name the channel already +# has is the one thing this must never do. +for f in "${FILES[@]}"; do + b=$(basename "$f") + grep -qxF "$b" <<<"$listing" || continue + remote_sum=$(rclone hashsum md5 "$DEST/$b" --s3-no-head 2>/dev/null | awk '{print $1}') + local_sum=$(md5sum "$f" | awk '{print $1}') + if [[ -n "$remote_sum" && "$remote_sum" == "$local_sum" ]]; then + print_info "Already published with identical bytes, adding to the database only: $b" + else + print_error "Already published with DIFFERENT bytes, refusing to overwrite: $b" + echo " Bump pkgrel; published filenames are immutable." + exit 1 + fi +done + +# --- 3. sign --------------------------------------------------------------- +export GNUPGHOME="$WORK/gnupg"; mkdir -m 700 "$GNUPGHOME" +echo "$GPG_PRIVATE_KEY" | gpg --batch --quiet --import +KEY_ID=$(gpg --list-secret-keys --with-colons | awk -F: '$1=="sec"{print $5; exit}') +[[ -n "$KEY_ID" ]] || { print_error "No secret key imported"; exit 1; } +for f in "${FILES[@]}"; do + cp "$f" "$WORK/repo/" + gpg --batch --yes --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" \ + --detach-sign --no-armor --local-user "$KEY_ID" "$WORK/repo/$(basename "$f")" + print_step "signed $(basename "$f")" +done + +# --- 4. repo-add (replaces the entry for each pkgname) --------------------- +( cd "$WORK/repo" && repo-add --quiet omarchy.db.tar.zst "${FILES[@]/#*\//}" ) +ln -sf omarchy.db.tar.zst "$WORK/repo/omarchy.db" +ln -sf omarchy.files.tar.zst "$WORK/repo/omarchy.files" +print_info "Database now has $(tar -tf "$WORK/repo/omarchy.db.tar.zst" | grep -c '/$') entries" + +# --- 5. upload: packages, signatures, database last ----------------------- +rclone copy "$WORK/repo" "$DEST" --s3-no-head --ignore-existing --filter '- *.sig' --filter '+ *.pkg.tar.*' --filter '- *' +rclone copy "$WORK/repo" "$DEST" --s3-no-head --ignore-existing --filter '+ *.pkg.tar.*.sig' --filter '- *' +# Re-verify every referenced file is really there before the db goes up. +listing=$(rclone lsf "$DEST/" --s3-no-head) +for f in "${FILES[@]}"; do + b=$(basename "$f") + grep -qxF "$b" <<<"$listing" && grep -qxF "$b.sig" <<<"$listing" || { print_error "Upload incomplete: $b"; exit 1; } +done +rclone copy "$WORK/repo" "$DEST" --s3-no-head --checksum --copy-links --filter '+ omarchy.db*' --filter '+ omarchy.files*' --filter '- *' +print_success "Published ${#FILES[@]} package(s) to $DEST" diff --git a/build/Dockerfile b/build/Dockerfile index af1bb5e..2a81a62 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -129,6 +129,7 @@ RUN pacman -Syu --noconfirm && \ wget \ curl \ jq \ + rclone \ gnupg && \ pacman -Scc --noconfirm && \ rm -rf /var/cache/pacman/pkg/* @@ -146,7 +147,8 @@ RUN useradd -m -G wheel -s /bin/bash builder && \ # be skipped at signing. Pin the extension so both architectures match. RUN sed -i 's/^#MAKEFLAGS=.*/MAKEFLAGS="-j$(nproc)"/' /etc/makepkg.conf && \ sed -i 's/^COMPRESSZST=.*/COMPRESSZST=(zstd -c -z -q --threads=0 -)/' /etc/makepkg.conf && \ - sed -i "s|^PKGEXT=.*|PKGEXT='.pkg.tar.zst'|" /etc/makepkg.conf + sed -i "s|^PKGEXT=.*|PKGEXT='.pkg.tar.zst'|" /etc/makepkg.conf && \ + sed -i 's|^#\?PACKAGER=.*|PACKAGER="Omarchy "|' /etc/makepkg.conf # Pacman wrapper that auto-resolves package conflicts (e.g. rustup vs rust). # makepkg only passes --noconfirm to pacman, which defaults to 'N' on conflict diff --git a/build/build.sh b/build/build.sh index 4c39147..65d6017 100755 --- a/build/build.sh +++ b/build/build.sh @@ -26,6 +26,29 @@ DEFER_RUNTIME_DEPS=${DEFER_RUNTIME_DEPS:-false} source "$HELPERS_DIR/package-metadata.sh" +# Where the channel's published database is read from for planning. On the +# repository host it is the published tree itself. Anywhere else (a CI runner, +# a fresh clone) that tree is absent, so the database is fetched from the +# public channel and the same URL serves as pacman's dependency repository. +# Set OMARCHY_PUBLISHED_REPO_URL= (empty) to disable the remote fallback. +PUBLISHED_REPO_URL=${OMARCHY_PUBLISHED_REPO_URL-https://pkgs.omarchy.org} +PUBLISHED_DB_DIR="$FINAL_OUTPUT_DIR" +PUBLISHED_REPO_SERVER="" +if [[ ! -f "$FINAL_OUTPUT_DIR/omarchy.db.tar.zst" && ! -f "$FINAL_OUTPUT_DIR/omarchy.db" && -n "$PUBLISHED_REPO_URL" ]]; then + remote_channel="$PUBLISHED_REPO_URL/$MIRROR/$ARCH" + remote_db_dir=$(mktemp -d /tmp/omarchy-published.XXXXXX) || exit 1 + # Cache-bust: the channel sits behind a CDN that serves a stale database + # for a while after a sync. + if curl -fsSL "$remote_channel/omarchy.db.tar.zst?$(date +%s)" -o "$remote_db_dir/omarchy.db.tar.zst"; then + PUBLISHED_DB_DIR="$remote_db_dir" + PUBLISHED_REPO_SERVER="$remote_channel" + echo "==> No local published tree; planning against $remote_channel" + else + rm -rf "$remote_db_dir" + echo "==> No local published tree and $remote_channel is unavailable; treating the channel as empty" + fi +fi + if [[ $DEFER_RUNTIME_DEPS != "false" && $DEFER_RUNTIME_DEPS != "true" ]]; then echo "DEFER_RUNTIME_DEPS must be true or false" >&2 exit 1 @@ -118,10 +141,15 @@ if [[ "$DRY_RUN" != true ]]; then fi touch "$BUILD_PLAN_DIR/repository-initialized" || exit 1 - # Add omarchy repo if it has a database (stable packages) + # Add omarchy repo if it has a database (stable packages). The local tree + # is trusted as-is; the public channel is verified against the omarchy + # keyring the image already carries. if [[ -f "$FINAL_OUTPUT_DIR/omarchy.db.tar.zst" ]] || [[ -f "$FINAL_OUTPUT_DIR/omarchy.db" ]]; then sudo sed -i "/^\[core\]$/i [omarchy]\nSigLevel = Optional TrustAll\nServer = file://$FINAL_OUTPUT_DIR\n" /etc/pacman.conf echo " -> omarchy (priority 2): $FINAL_OUTPUT_DIR" + elif [[ -n "$PUBLISHED_REPO_SERVER" ]]; then + sudo sed -i "/^\[core\]$/i [omarchy]\nSigLevel = Required DatabaseOptional\nServer = $PUBLISHED_REPO_SERVER\n" /etc/pacman.conf + echo " -> omarchy (priority 2): $PUBLISHED_REPO_SERVER" fi # Sync pacman database @@ -159,10 +187,10 @@ LOCAL_VERSION_CACHE_LOADED=false LOCAL_VERSION_CACHE_DB="" load_local_versions() { - local db="$FINAL_OUTPUT_DIR/omarchy.db.tar.zst" + local db="$PUBLISHED_DB_DIR/omarchy.db.tar.zst" if [[ ! -f "$db" ]]; then - db="$FINAL_OUTPUT_DIR/omarchy.db" + db="$PUBLISHED_DB_DIR/omarchy.db" fi [[ -f "$db" ]] || return 0 diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..0e53988 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,79 @@ +# CI spike: build PRs on ephemeral DigitalOcean droplets + +Status: spike. Nothing here publishes. The repository host keeps building and +signing on merge exactly as before. + +## Pieces + +- `.github/workflows/build-pr.yml` — on a PR touching `pkgbuilds/**`, one job + per changed package on runners labelled `omarchy-builder`. Uploads the + unsigned `.pkg.tar.zst` as a workflow artifact (7 days). +- `runner-cloud-init.yaml` — Ubuntu 24.04 user-data: docker + buildx, the + GitHub runner registered `--ephemeral`, runs one job, powers off. +- `controller.sh` — systemd timer every minute on a small always-on droplet. + Polls for queued jobs with our label, creates one g5-32vcpu-64gb-50gb droplet (ric1) per job up + to `MAX_DROPLETS`, deletes droplets that are powered off or older than + `MAX_AGE_MINUTES`. No inbound endpoint. Plain curl against both APIs, no + doctl and no gh: a token in the environment cannot pick the wrong account + the way a saved doctl context can. Needs curl and jq. + `tests/controller.sh` exercises every decision against canned responses. +- `controller-box/` — the always-on droplet: unit, timer, env template, + cloud-init, and `create.sh` to stand it up with one API call. + +## Standing up the controller box + + DIGITALOCEAN_TOKEN= GITHUB_TOKEN= \ + REPO=omacom/omarchy-pkgs ci/controller-box/create.sh + +The GitHub PAT is fine-grained, scoped to the one repo: Actions read, +Administration read+write (registration tokens). The DO token is baked into +the box's env file, so it is the account that pays for builder droplets. +Watch it with `journalctl -u omarchy-controller -f` on the box. + +## What the spike proved (2026-09-17, fork ryanrhughes/omarchy-pkgs) + +- `bin/build` works from a bare clone: with no local published tree it + plans against and resolves from `https://pkgs.omarchy.org//`. +- Droplet create → runner registered: ~70 s. omarchy-fish PR job: 2 min + including the builder image build. Droplet powers off after the job. +- linux-omarchy on a c-32 droplet: 30 min wall clock for the build job + (23:39 → 00:09), 254 MB artifact. Cold start ~90 s before the job began. +- A PR whose PKGBUILD fails to build turns the required check red and GitHub + refuses the merge (`mergeStateStatus=BLOCKED`, `gh pr merge` refuses + without `--admin`). +- Controller: one queued job + one busy droplet ⇒ creates exactly one more; + reaps powered-off droplets on the next tick. + +## Not done (required before this touches the real repo) + +- Tooling from base: check out master's `bin/ helpers/ build/` and overlay + only the PR's `pkgbuilds/`; today a PR can edit the build script + and it runs on the droplet. The vouch gate limits who can do that, not + what they can do. +- DigitalOcean cloud firewall on the `omarchy-builder` tag: no inbound, no + egress to private ranges or the metadata address. +- A fine-grained GitHub token for the real repository (the one on the + controller box is scoped to the fork), and the publish environment's + secrets set there. +- Disable the host's auto-release timers for any channel CI publishes to, + so two writers never touch one database. + +## Done since the spike README was first written + +- Controller as a systemd timer on its own droplet, plain curl, self-test. +- Build once against edge; one artifact per package per architecture, + published into every channel it belongs to (fast ring: all three at + once). arch=any builds once for every architecture database. +- Publish is incremental and immutable: pull the channel db, refuse + different bytes under an existing name, accept identical bytes, upload + packages then signatures then the db. +- aarch64 under QEMU with credential-preserving binfmt. +- Vouch gate: collaborators, `.github/VOUCHED.td`, or the `build-approved` + label; denounced authors cannot be overridden by the label. +- Tests run on PRs only; `result`, `self-tests`, `build-isolation` are the + required checks with strict up-to-date branches. + +## Cleanup + + doctl compute droplet list --tag-name omarchy-builder + doctl compute droplet delete -f diff --git a/ci/controller-box/cloud-init.yaml b/ci/controller-box/cloud-init.yaml new file mode 100644 index 0000000..fda8c43 --- /dev/null +++ b/ci/controller-box/cloud-init.yaml @@ -0,0 +1,45 @@ +#cloud-config +# The always-on controller droplet (smallest size is fine). Clones the repo +# for ci/controller.sh, installs the unit and timer, and starts polling. +# +# Substitute before use: +# __REPO_URL__ https://github.com/omacom/omarchy-pkgs.git +# __BRANCH__ branch carrying ci/ (master once merged) +# __ENV_B64__ base64 of a filled-in controller.env.example +# __SSH_KEYS_JSON__ JSON array of public keys authorized for root +package_update: true +packages: [curl, jq, git] + +# Root stays reachable by key so the journal can be read. Two things stand +# in the way on DO images: disable_root rewrites root's keys into a stub, and +# with no account ssh key attached DO expires root's password, which makes +# sshd refuse every non-interactive session with "password change required". +disable_root: false +chpasswd: + expire: false +ssh_authorized_keys: __SSH_KEYS_JSON__ + +users: + - name: controller + shell: /bin/bash + +write_files: + # defer: write after the users module has created the controller group, + # otherwise chown to root:controller fails and the unit cannot read this. + - path: /etc/omarchy-controller.env + permissions: "0640" + owner: root:controller + encoding: b64 + defer: true + content: __ENV_B64__ + +runcmd: + - chage -d "$(date +%F)" -M -1 root + - chown root:controller /etc/omarchy-controller.env && chmod 0640 /etc/omarchy-controller.env + - git clone --depth 1 --branch __BRANCH__ __REPO_URL__ /opt/omarchy-pkgs + - mkdir -p /run/omarchy-controller && chown controller:controller /run/omarchy-controller + - echo "d /run/omarchy-controller 0755 controller controller -" > /etc/tmpfiles.d/omarchy-controller.conf + # runcmd is executed by /bin/sh: no brace expansion. + - cp /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.service /opt/omarchy-pkgs/ci/controller-box/omarchy-controller.timer /etc/systemd/system/ + - systemctl daemon-reload + - systemctl enable --now omarchy-controller.timer diff --git a/ci/controller-box/controller.env.example b/ci/controller-box/controller.env.example new file mode 100644 index 0000000..37ae372 --- /dev/null +++ b/ci/controller-box/controller.env.example @@ -0,0 +1,15 @@ +# /etc/omarchy-controller.env — mode 0600, owned by root, read by systemd. +DIGITALOCEAN_TOKEN=dop_v1_... +# Fine-grained PAT scoped to the repo: Actions: read, Administration: read+write +GITHUB_TOKEN=github_pat_... +REPO=omacom/omarchy-pkgs +LABEL=omarchy-builder +TAG=omarchy-builder +REGION=ric1 +SIZE=g5-32vcpu-64gb-50gb +MAX_DROPLETS=6 +MAX_AGE_MINUTES=200 +LOCK=/run/omarchy-controller/lock +# Operator public keys for root on every builder droplet (JSON array). +# create.sh fills this from the operators' GitHub keys. +SSH_KEYS_JSON=[] diff --git a/ci/controller-box/create.sh b/ci/controller-box/create.sh new file mode 100755 index 0000000..9ec4c3d --- /dev/null +++ b/ci/controller-box/create.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Create the controller droplet with plain curl. Run from a laptop, once. +# +# DIGITALOCEAN_TOKEN=... GITHUB_TOKEN=... ci/controller-box/create.sh [branch] +# +# The DO token given here is baked into the box's env file, so it must be the +# token for the account that should pay for builder droplets. +set -euo pipefail +here=$(dirname "$0") +: "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}" +REPO=${REPO:-omacom/omarchy-pkgs} +BRANCH=${1:-master} +REGION=${REGION:-ric1} +NAME=${NAME:-omarchy-controller} +# Optional DO ssh key ids as a JSON array, e.g. SSH_KEYS='[123]', for reading +# the journal while bringing the box up. Not needed once it works. +SSH_KEYS=${SSH_KEYS:-[]} +# Public keys authorized for root: the operators' GitHub keys, fetched at +# creation so the box never depends on an ssh_key API scope. Override with +# ADMIN_GITHUB_USERS. +ADMIN_GITHUB_USERS=${ADMIN_GITHUB_USERS:-ryanrhughes dhh} +ssh_keys_json=$(for u in $ADMIN_GITHUB_USERS; do curl -fsS "https://github.com/$u.keys"; done | jq -R . | jq -sc .) +[[ $(jq length <<<"$ssh_keys_json") -gt 0 ]] || { echo "no ssh keys fetched for $ADMIN_GITHUB_USERS" >&2; exit 1; } + +env_file=$(sed -e "s|^DIGITALOCEAN_TOKEN=.*|DIGITALOCEAN_TOKEN=$DIGITALOCEAN_TOKEN|" \ + -e "s|^GITHUB_TOKEN=.*|GITHUB_TOKEN=$GITHUB_TOKEN|" \ + -e "s|^REPO=.*|REPO=$REPO|" \ + -e "s|^SSH_KEYS_JSON=.*|SSH_KEYS_JSON=$ssh_keys_json|" "$here/controller.env.example") +userdata=$(sed -e "s|__REPO_URL__|https://github.com/$REPO.git|" -e "s|__BRANCH__|$BRANCH|" \ + -e "s|__ENV_B64__|$(printf '%s\n' "$env_file" | base64 -w0)|" \ + -e "s|__SSH_KEYS_JSON__|$ssh_keys_json|" "$here/cloud-init.yaml") +body=$(jq -n --arg name "$NAME" --arg region "$REGION" --arg ud "$userdata" --argjson keys "$SSH_KEYS" \ + '{name:$name, region:$region, size:"s-1vcpu-1gb", image:"ubuntu-24-04-x64", tags:["omarchy-controller"], user_data:$ud, ssh_keys:$keys}') + +# Refuse to create a second one. +existing=$(curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ + "https://api.digitalocean.com/v2/droplets?tag_name=omarchy-controller" | jq '.droplets | length') +if (( existing > 0 )); then echo "a controller droplet already exists" >&2; exit 1; fi + +curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" -H "Content-Type: application/json" \ + -X POST -d "$body" https://api.digitalocean.com/v2/droplets | jq -r '"created \(.droplet.name) id=\(.droplet.id)"' diff --git a/ci/controller-box/omarchy-controller.service b/ci/controller-box/omarchy-controller.service new file mode 100644 index 0000000..12d2e9c --- /dev/null +++ b/ci/controller-box/omarchy-controller.service @@ -0,0 +1,12 @@ +[Unit] +Description=Provision ephemeral omarchy-builder runner droplets for queued jobs +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=controller +EnvironmentFile=/etc/omarchy-controller.env +ExecStart=/opt/omarchy-pkgs/ci/controller.sh +# The reaper's safety net is time, not state; a hung tick must not hold the lock. +TimeoutStartSec=240 diff --git a/ci/controller-box/omarchy-controller.timer b/ci/controller-box/omarchy-controller.timer new file mode 100644 index 0000000..0534b68 --- /dev/null +++ b/ci/controller-box/omarchy-controller.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run the omarchy-builder controller every minute + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/ci/controller.sh b/ci/controller.sh new file mode 100755 index 0000000..cc60950 --- /dev/null +++ b/ci/controller.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Droplet-per-job controller for the omarchy-builder runner pool. +# +# Run from a systemd timer every minute on a small always-on droplet. No +# inbound endpoint: it polls GitHub for queued jobs wanting our label, creates +# one ephemeral droplet per job (up to MAX_DROPLETS), and deletes droplets +# that have powered off or exceeded MAX_AGE_MINUTES. The reaper does not +# trust its own bookkeeping: it lists by tag and acts on what DigitalOcean +# reports. +# +# Talks to both APIs with curl. No doctl: its saved contexts silently choose +# an account; a token in the environment cannot. Needs curl and jq. +# +# Environment: +# DIGITALOCEAN_TOKEN DO API token for the account that pays for droplets +# GITHUB_TOKEN fine-grained PAT: Actions read, Administration write +# REPO owner/name +set -euo pipefail + +REPO=${REPO:?owner/name} +: "${DIGITALOCEAN_TOKEN:?}" "${GITHUB_TOKEN:?}" +LABEL=${LABEL:-omarchy-builder} +TAG=${TAG:-omarchy-builder} +REGION=${REGION:-ric1} +SIZE=${SIZE:-g5-32vcpu-64gb-50gb} +IMAGE=${IMAGE:-ubuntu-24-04-x64} +MAX_DROPLETS=${MAX_DROPLETS:-4} +MAX_AGE_MINUTES=${MAX_AGE_MINUTES:-200} +RUNNER_VERSION=${RUNNER_VERSION:-2.337.0} +CLOUD_INIT=${CLOUD_INIT:-$(dirname "$0")/runner-cloud-init.yaml} +# Operator public keys authorized on every builder (JSON array of strings). +# The box's env file carries them; empty means no root login. +SSH_KEYS_JSON=${SSH_KEYS_JSON:-[]} +LOCK=${LOCK:-/tmp/omarchy-controller.lock} + +log() { echo "$(date '+%F %T') $*"; } + +# The only two places the outside world is touched. The self-test overrides +# both, so every decision below is exercised against canned responses. +do_api() { # do_api [curl args...] + local path=$1; shift + curl -fsS -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ + -H "Content-Type: application/json" "https://api.digitalocean.com/v2/$path" "$@" +} +gh_api() { # gh_api [curl args...] + local path=$1; shift + curl -fsS -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" "https://api.github.com/$path" "$@" +} + +# --- reap ------------------------------------------------------------------ +reap() { + local now id status created age + now=$(date +%s) + while read -r id status created; do + [[ -n "$id" ]] || continue + age=$(( (now - $(date -d "$created" +%s)) / 60 )) + if [[ $status == off ]] || (( age > MAX_AGE_MINUTES )); then + log "deleting droplet $id (status=$status age=${age}m)" + do_api "droplets/$id" -X DELETE + fi + done < <(do_api "droplets?tag_name=$TAG&per_page=200" | + jq -r '.droplets[] | "\(.id) \(.status) \(.created_at)"') +} + +# --- demand ---------------------------------------------------------------- +queued_jobs() { + local run + gh_api "repos/$REPO/actions/runs?status=queued&per_page=50" --get \ + | jq -r '.workflow_runs[].id' | + while read -r run; do + gh_api "repos/$REPO/actions/runs/$run/jobs" \ + | jq -r --arg l "$LABEL" '.jobs[] | select(.status=="queued") | select(.labels | index($l)) | .id' + done | wc -l +} + +live_droplets() { + do_api "droplets?tag_name=$TAG&per_page=200" | jq '[.droplets[] | select(.status != "off")] | length' +} + +busy_runners() { + gh_api "repos/$REPO/actions/runners?per_page=100" \ + | jq --arg l "$LABEL" '[.runners[] | select(.busy) | select(any(.labels[]; .name == $l))] | length' +} + +# --- create ---------------------------------------------------------------- +create_droplet() { + local token userdata name body + token=$(gh_api "repos/$REPO/actions/runners/registration-token" -X POST | jq -r .token) + userdata=$(sed -e "s|__REPO__|$REPO|g" -e "s|__RUNNER_TOKEN__|$token|g" \ + -e "s|__RUNNER_LABELS__|$LABEL|g" -e "s|__RUNNER_VERSION__|$RUNNER_VERSION|g" \ + -e "s|__SSH_KEYS_JSON__|$SSH_KEYS_JSON|" "$CLOUD_INIT") + name="$TAG-$(date +%s)-$RANDOM" + body=$(jq -n --arg name "$name" --arg region "$REGION" --arg size "$SIZE" --arg image "$IMAGE" \ + --arg tag "$TAG" --arg ud "$userdata" \ + '{name:$name, region:$region, size:$size, image:$image, tags:[$tag], user_data:$ud, monitoring:false}') + log "creating $name ($SIZE)" + do_api droplets -X POST -d "$body" | jq -r '"created droplet \(.droplet.id)"' +} + +controller_tick() { + reap + local queued live busy available need room + queued=$(queued_jobs) + live=$(live_droplets) + busy=$(busy_runners) + # A live droplet whose runner is busy is spoken for. Only droplets still + # booting or listening can absorb a queued job. + available=$(( live - busy )); (( available < 0 )) && available=0 + need=$(( queued - available )) + (( need > 0 )) || return 0 + room=$(( MAX_DROPLETS - live )) + (( need > room )) && need=$room + if (( need <= 0 )); then + log "at cap ($live/$MAX_DROPLETS, $busy busy) with $queued queued" + return 0 + fi + local i + for (( i = 0; i < need; i++ )); do create_droplet; done +} + +if [[ "${CONTROLLER_LIBRARY_ONLY:-}" != 1 ]]; then + exec 9>"$LOCK"; flock -n 9 || exit 0 + controller_tick +fi diff --git a/ci/runner-cloud-init.yaml b/ci/runner-cloud-init.yaml new file mode 100644 index 0000000..c2db181 --- /dev/null +++ b/ci/runner-cloud-init.yaml @@ -0,0 +1,77 @@ +#cloud-config +# Ephemeral GitHub Actions runner for omarchy-pkgs package builds. +# +# Boots an Ubuntu droplet, installs docker and the runner, registers ONCE with +# --ephemeral, runs exactly one job, then powers off. The controller (or the +# reaper) deletes the powered-off droplet. Nothing here holds a long-lived +# credential: the registration token is single-use and expires in an hour. +# +# Substitute before use: +# __REPO__ owner/name +# __RUNNER_TOKEN__ registration token (gh api -X POST repos/O/R/actions/runners/registration-token) +# __RUNNER_LABELS__ e.g. omarchy-builder +# __RUNNER_VERSION__ e.g. 2.329.0 + +# Operators can reach a builder by key while it lives; it powers off after +# one job anyway. Keys are substituted by the controller (__SSH_KEYS_JSON__). +disable_root: false +chpasswd: + expire: false +ssh_authorized_keys: __SSH_KEYS_JSON__ + +package_update: true +packages: + - docker.io + - docker-buildx + - unzip + - git + - curl + - jq + - rsync + +users: + - name: runner + groups: [docker] + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + +write_files: + # defer: write after users/groups exist, so /home/runner is created by + # useradd (owned by runner) rather than by this module as root. + - path: /home/runner/start.sh + permissions: "0755" + owner: runner:runner + defer: true + content: | + #!/bin/bash + set -euo pipefail + cd /home/runner + mkdir -p actions-runner && cd actions-runner + arch=$(uname -m); [[ $arch == x86_64 ]] && arch=x64 + curl -fsSL -o runner.tgz \ + "https://github.com/actions/runner/releases/download/v__RUNNER_VERSION__/actions-runner-linux-${arch}-__RUNNER_VERSION__.tar.gz" + tar xzf runner.tgz && rm runner.tgz + ./config.sh --unattended --ephemeral \ + --url "https://github.com/__REPO__" \ + --token "__RUNNER_TOKEN__" \ + --name "do-$(hostname)" \ + --labels "__RUNNER_LABELS__" \ + --replace + ./run.sh + # One job done. Power off; the controller deletes powered-off droplets. + sudo poweroff + +runcmd: + # With no account ssh key attached, DO expires root's password, and sshd + # then refuses every non-interactive session. Clear it first so operators + # can read the logs of a builder that never registers. + - chage -d "$(date +%F)" -M -1 root + - systemctl enable --now docker + # aarch64 builds run under user-mode emulation (DO has no arm droplets). + # Register QEMU with the F and C flags via the multiarch image, exactly as + # helpers/docker-helpers.sh setup_qemu does: Ubuntu's qemu-user-static + # package registers without C, so sudo inside the emulated container fails + # with "effective uid is not 0". Best-effort: an x86-only job never needs it. + - docker run --rm --privileged docker.io/multiarch/qemu-user-static --reset -p yes --credential yes || true + - chown -R runner:runner /home/runner + - sudo -u runner /home/runner/start.sh > /home/runner/runner.log 2>&1 diff --git a/tests/controller.sh b/tests/controller.sh new file mode 100755 index 0000000..d777fc5 --- /dev/null +++ b/tests/controller.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Self-test for ci/controller.sh: every decision, no cloud. +# +# The controller's two API functions are overridden with canned responses and +# a recorder, then each scenario asserts which creates and deletes it issued. +set -euo pipefail +ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") + +export REPO=o/r DIGITALOCEAN_TOKEN=x GITHUB_TOKEN=x +export CLOUD_INIT="$ROOT/ci/runner-cloud-init.yaml" LOCK=/tmp/controller-test.lock +CONTROLLER_LIBRARY_ONLY=1 source "$ROOT/ci/controller.sh" + +# Calls are recorded to a file: the controller invokes the API functions +# inside command substitutions, and a subshell cannot append to an array. +CALLS_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE"' EXIT +NOW=$(date -u +%FT%TZ) +OLD=$(date -u -d '5 hours ago' +%FT%TZ) + +# Scenario state: DROPLETS is "id status created" lines, QUEUED a count, +# BUSY a count. +do_api() { + local path=$1; shift + echo "do $path $*" >>"$CALLS_FILE" + case "$path" in + droplets\?*) printf '%s\n' "$DROPLETS" | jq -Rs '{droplets: [split("\n")[] | select(length>0) | split(" ") | {id: .[0]|tonumber, status: .[1], created_at: .[2]}]}' ;; + droplets) echo '{"droplet":{"id":999}}' ;; + droplets/*) echo '{}' ;; + esac +} +gh_api() { + local path=$1; shift + echo "gh $path $*" >>"$CALLS_FILE" + case "$path" in + */actions/runs\?*) jq -nc --argjson n "$QUEUED" '{workflow_runs: [range($n) | {id: .}]}' ;; + */actions/runs/*/jobs) echo '{"jobs":[{"id":1,"status":"queued","labels":["self-hosted","omarchy-builder"]}]}' ;; + */actions/runners\?*) jq -nc --argjson n "$BUSY" '{runners: [range($n) | {busy: true, labels: [{name: "omarchy-builder"}]}]}' ;; + */registration-token) echo '{"token":"T"}' ;; + esac +} + +creates() { grep -c '^do droplets -X POST' "$CALLS_FILE" || true; } +deletes() { grep -c '^do droplets/.* -X DELETE' "$CALLS_FILE" || true; } +run() { : >"$CALLS_FILE"; controller_tick >/dev/null; } +check() { # check + local c d; c=$(creates); d=$(deletes) + if [[ "$c" == "$2" && "$d" == "$3" ]]; then echo "PASS: $1"; else echo "FAIL: $1 (creates=$c want $2, deletes=$d want $3)"; cat "$CALLS_FILE"; exit 1; fi +} + +DROPLETS="" QUEUED=0 BUSY=0; run; check "idle: nothing queued, nothing to reap" 0 0 +DROPLETS="" QUEUED=2 BUSY=0; run; check "two queued, none live: create two" 2 0 +DROPLETS="1 active $NOW" QUEUED=1 BUSY=1; run; check "one queued, one live but busy: create one" 1 0 +DROPLETS="1 active $NOW" QUEUED=1 BUSY=0; run; check "one queued, one live and idle: it will take it" 0 0 +DROPLETS="1 off $NOW" QUEUED=0 BUSY=0; run; check "powered-off droplet reaped" 0 1 +DROPLETS="1 active $OLD" QUEUED=0 BUSY=0; run; check "over-age droplet reaped even if active" 0 1 +DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW"$'\n3 active '"$NOW"$'\n4 active '"$NOW" QUEUED=3 BUSY=4; MAX_DROPLETS=4; run; check "at cap: no creates" 0 0 +DROPLETS=$'1 active '"$NOW"$'\n2 active '"$NOW" QUEUED=5 BUSY=2; MAX_DROPLETS=3; run; check "cap limits creates to remaining room" 1 0 +DROPLETS="1 off $NOW" QUEUED=1 BUSY=0; MAX_DROPLETS=4; run; check "off droplet is not capacity: reaped and replaced" 1 1 + +# The create body must carry the tag (reaper scope) and substituted user-data. +BODY_FILE=$(mktemp); trap 'rm -f "$CALLS_FILE" "$BODY_FILE"' EXIT +do_api() { if [[ $1 == droplets ]]; then printf '%s' "${*: -1}" >"$BODY_FILE"; echo '{"droplet":{"id":1}}'; else echo '{"droplets":[]}'; fi; } +gh_api() { echo '{"token":"TOK"}'; } +create_droplet >/dev/null +jq -e '.tags == ["omarchy-builder"] and .size == "g5-32vcpu-64gb-50gb" and (.user_data | test("--token \"TOK\"")) and (.user_data | test("__") | not)' "$BODY_FILE" >/dev/null \ + && echo "PASS: create body carries tag, size, substituted user-data" \ + || { echo "FAIL: create body"; jq . "$BODY_FILE" | head -20; exit 1; } diff --git a/tests/publish-artifact.sh b/tests/publish-artifact.sh new file mode 100755 index 0000000..2c5269c --- /dev/null +++ b/tests/publish-artifact.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Self-test for bin/publish-artifact against a local directory as the remote. +# Needs repo-add, gpg, rclone, bsdtar (run in the Arch builder/test container). +set -euo pipefail +ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +T=$(mktemp -d); chmod 755 "$T"; trap 'rm -rf "$T"' EXIT +REMOTE="$T/r2"; mkdir -p "$REMOTE" + +# throwaway signing key +export GNUPGHOME="$T/g"; mkdir -m700 "$GNUPGHOME" +gpg --batch --quiet --passphrase '' --quick-gen-key 'Test ' ed25519 sign 0 2>/dev/null +export GPG_PRIVATE_KEY=$(gpg --batch --armor --export-secret-keys 'Test ') GPG_PASSPHRASE='' +unset GNUPGHOME + +# minimal real packages via makepkg +mkpkg() { # mkpkg [payload] + local d="$T/src/$1-$2${4:+-$4}"; mkdir -p "$d"; cd "$d" + printf 'pkgname=%s\npkgver=1.0\npkgrel=%s\narch=(%s)\npackage(){ install -Dm644 /dev/null "$pkgdir/usr/share/%s-%s"; echo "%s" > "$pkgdir/usr/share/%s-%s"; }\n' "$1" "$2" "$3" "$1" "$2" "${4:-payload}" "$1" "$2" > PKGBUILD + # CARCH so the PKGINFO records the requested arch (--ignorearch would + # stamp the host's). + # makepkg refuses to run as root (the CI test container does); build the + # fixture as an unprivileged user in that case. + if (( EUID == 0 )); then + id -u fixture >/dev/null 2>&1 || useradd -m fixture + chmod 755 "$T/src"; chown -R fixture "$d" + runuser -u fixture -- env CARCH=$3 makepkg -f --nodeps --ignorearch >/dev/null 2>&1 + else + CARCH=$3 makepkg -f --nodeps --ignorearch >/dev/null 2>&1 + fi + ls "$d"/*.pkg.tar.zst +} +A1=$(mkpkg alpha 1 any); A2=$(mkpkg alpha 2 any); B1=$(mkpkg beta 1 x86_64); C1=$(mkpkg gamma 1 aarch64) + +pub() { "$ROOT/bin/publish-artifact" --remote "$REMOTE" --mirror edge --arch x86_64 "$@" >"$T/out" 2>&1; } +entries() { tar -tf "$REMOTE/edge/x86_64/omarchy.db.tar.zst" | grep '/$' | sort | tr '\n' ' '; } +pass() { echo "PASS: $1"; } +fail() { echo "FAIL: $1"; cat "$T/out"; exit 1; } + +pub "$A1" && [[ "$(entries)" == "alpha-1.0-1/ " ]] && [[ -f "$REMOTE/edge/x86_64/$(basename "$A1").sig" ]] \ + && pass "first publish creates db with one entry and a signature" || fail "first publish" + +sum_before=$(sha256sum "$REMOTE/edge/x86_64/$(basename "$A1")") +pub "$B1" && [[ "$(entries)" == "alpha-1.0-1/ beta-1.0-1/ " ]] && [[ "$(sha256sum "$REMOTE/edge/x86_64/$(basename "$A1")")" == "$sum_before" ]] \ + && pass "second package added incrementally; first file untouched" || fail "incremental add" + +pub "$A2" && [[ "$(entries)" == "alpha-1.0-2/ beta-1.0-1/ " ]] && [[ -f "$REMOTE/edge/x86_64/$(basename "$A1")" ]] \ + && pass "new pkgrel replaces the db entry, old file remains on remote" || fail "replace entry" + +# Same bytes again: allowed, idempotent (this is how a fast-ring artifact +# reaches rc and stable after edge, and how a re-run recovers). +pub "$A2" && grep -q 'identical bytes' "$T/out" && [[ "$(entries)" == "alpha-1.0-2/ beta-1.0-1/ " ]] \ + && pass "identical bytes under an existing name: accepted, db unchanged" || fail "identical republish" + +# Orphan repair: a file that reached the remote but whose db entry was lost +# (a concurrent publish overwrote the db) is fixed by publishing it again. +( cd "$REMOTE/edge/x86_64" && repo-remove --quiet omarchy.db.tar.zst alpha >/dev/null 2>&1 ) +[[ "$(entries)" == "beta-1.0-1/ " ]] || fail "fixture: could not drop alpha from the db" +pub "$A2" && [[ "$(entries)" == "alpha-1.0-2/ beta-1.0-1/ " ]] \ + && pass "orphaned file regains its db entry on republish" || fail "orphan repair" + +# Different bytes under an existing name: refused. Build alpha-2 again with +# a different payload (makepkg is reproducible, so the content must change). +A2b=$(mkpkg alpha 2 any different-payload) +[[ "$(md5sum < "$A2")" != "$(md5sum < "$A2b")" ]] || { echo "fixture: rebuilt package is byte-identical, cannot test"; exit 1; } +if pub "$A2b"; then fail "different bytes under same filename should refuse"; else grep -q 'DIFFERENT bytes' "$T/out" && pass "different bytes under an existing name refused" || fail "wrong refusal reason"; fi + +if pub "$C1"; then fail "aarch64 package into x86_64 should refuse"; else grep -q 'publishing to x86_64' "$T/out" && pass "wrong-arch package refused" || fail "wrong-arch reason"; fi + +cp "$B1" "$T/renamed-1.0-1-x86_64.pkg.tar.zst" +if pub "$T/renamed-1.0-1-x86_64.pkg.tar.zst"; then fail "filename/PKGINFO mismatch should refuse"; else grep -q 'does not match PKGINFO' "$T/out" && pass "filename must match PKGINFO" || fail "mismatch reason"; fi + +# db must verify: pacman can read it and each package's signature checks +gpg --batch --quiet --import <<<"$GPG_PRIVATE_KEY" 2>/dev/null || true +( cd "$REMOTE/edge/x86_64" && for f in *.pkg.tar.zst; do gpg --batch --quiet --verify "$f.sig" "$f" 2>/dev/null || { echo "FAIL: signature $f"; exit 1; }; done ) && pass "all signatures verify"