Emir Beganovic
5bd85a785f
A PR can change a package's directory without bumping its version: an upstream hook, its tests, a README. bin/build then plans nothing, and the pack step failed on the empty build output, turning the required result check red (seen on #769). Make the same dry-run check publish.yml's rebuild job already makes, and skip packing and uploading when nothing was built. publish.yml finds no artifact for such a merge and records the package as already published.
308 lines
17 KiB
YAML
308 lines
17 KiB
YAML
name: Build changed packages
|
|
|
|
# Build every package directory a PR touches, one job per package per arch:
|
|
# x86_64 on the self-hosted droplet pool, aarch64 natively on GitHub's arm64
|
|
# runners. Artifacts are unsigned; publish.yml signs and publishes them on
|
|
# merge.
|
|
#
|
|
# Tooling runs from the base branch; a PR supplies only pkgbuilds/. The
|
|
# vouch gate limits who may spend compute; this limits what their PR can run.
|
|
|
|
# No paths filter: approved PRs must report the required `result` even when
|
|
# no package directory changed. Those PRs get an empty matrix and a passing
|
|
# result in seconds; unapproved PRs wait for maintainer approval.
|
|
on:
|
|
pull_request:
|
|
types: [opened, synchronize, reopened, labeled]
|
|
workflow_dispatch:
|
|
inputs:
|
|
packages:
|
|
description: "Space-separated package directories to build"
|
|
required: true
|
|
|
|
# A new push normally cancels the PR's in-flight build. The sync bots'
|
|
# branches (auto/sync-*) are the exception: they are force-pushed with fresh
|
|
# upstream releases several times a day, which kept cancelling multi-hour
|
|
# aarch64 builds before they could finish. There the newest run waits
|
|
# instead (GitHub keeps at most one pending run per group, replacing older
|
|
# pending ones), and when it starts it reuses every artifact the finished
|
|
# build uploaded, so only packages whose tree changed are built again.
|
|
concurrency:
|
|
group: build-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: ${{ !(github.event.pull_request.head.repo.full_name == github.repository && startsWith(github.head_ref, 'auto/sync-')) }}
|
|
|
|
jobs:
|
|
# Builds cost real machines, so they run only for trusted authors:
|
|
# collaborators, anyone in .github/VOUCHED.td (read from the default
|
|
# branch, so a PR cannot vouch for itself), or a PR a maintainer has
|
|
# labelled "build-approved". Everyone else gets this job's plan output
|
|
# while the required `result` stays pending until a maintainer approves.
|
|
changes:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
matrix: ${{ steps.list.outputs.matrix }}
|
|
count: ${{ steps.gate.outputs.count }}
|
|
trusted: ${{ steps.gate.outputs.trusted }}
|
|
vouch_status: ${{ steps.vouch.outputs.status }}
|
|
empty: ${{ steps.list.outputs.empty }}
|
|
steps:
|
|
# Same rule as the build job: bin/build-matrix comes from the base
|
|
# branch tip, the package directories from the PR head.
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.event.pull_request.base.ref || github.sha }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
- if: github.event_name == 'pull_request'
|
|
run: |
|
|
git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.head.sha }}"
|
|
git checkout "${{ github.event.pull_request.head.sha }}" -- pkgbuilds/
|
|
# Bootstrap: the PR that introduces this tooling has a base without
|
|
# it. Take the plan helper from the PR head in that one case; it
|
|
# runs on a hosted runner and only prints a plan.
|
|
if [[ ! -x bin/build-matrix ]]; then
|
|
git checkout "${{ github.event.pull_request.head.sha }}" -- bin/build-matrix helpers/
|
|
echo "::notice::base branch has no bin/build-matrix; using the PR's copy for planning"
|
|
fi
|
|
- id: vouch
|
|
if: github.event_name == 'pull_request'
|
|
uses: mitchellh/vouch/action/check-user@f23dbb5e745334f97414ec70463ce7301071a661 # v1
|
|
with:
|
|
user: ${{ github.event.pull_request.user.login }}
|
|
allow-fail: true
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
- id: approval
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const { data: pr } = await github.rest.pulls.get({
|
|
...context.repo, pull_number: context.payload.pull_request.number,
|
|
});
|
|
// Approving or rerunning a held run keeps its original event,
|
|
// which may predate the label. Read the current approval instead.
|
|
core.setOutput('approved', pr.state === 'open' &&
|
|
pr.head.sha === context.payload.pull_request.head.sha &&
|
|
pr.labels.some(label => label.name === 'build-approved'));
|
|
# One matrix entry per package per architecture. Every package builds
|
|
# once, against edge; the channels it ships to on merge are carried
|
|
# along for information. A filename means one set of bytes.
|
|
- id: list
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
if [[ -n "${{ github.event.inputs.packages }}" ]]; then
|
|
names="${{ github.event.inputs.packages }}"
|
|
else
|
|
# The PR's own files, as GitHub lists them against the merge base.
|
|
# A two-dot diff against the current base tip also counted every
|
|
# package master changed after the PR branched, so a stale PR
|
|
# planned dozens of unrelated packages at its old versions. The
|
|
# checkout here is shallow, so there is no merge base to diff from.
|
|
# A package the PR deletes has nothing to build.
|
|
names=$(gh api --paginate "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" --jq '.[].filename' \
|
|
| awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u \
|
|
| while read -r name; do
|
|
if git cat-file -e "${{ github.event.pull_request.head.sha }}:pkgbuilds/$name" 2>/dev/null; then echo "$name"; fi
|
|
done)
|
|
fi
|
|
matrix=$(printf '%s\n' $names | bin/build-matrix)
|
|
# A package directory whose exact tree already has a build artifact
|
|
# (label <pkg>-<arch>-<tree hash>, uploaded only after a successful
|
|
# build) is not built again. Pushing a fix for one package to a PR
|
|
# that touches fifty rebuilds one, not fifty; publish.yml finds the
|
|
# same artifacts on merge. workflow_dispatch is an explicit request
|
|
# and always builds.
|
|
if [[ "${{ github.event_name }}" == pull_request ]]; then
|
|
head="${{ github.event.pull_request.head.sha }}"
|
|
kept=(); reused=()
|
|
while read -r entry; do
|
|
package=$(jq -r .package <<<"$entry"); arch=$(jq -r .arch <<<"$entry")
|
|
label="$package-$arch-$(git rev-parse "$head:pkgbuilds/$package")"
|
|
found=$(curl -fsS -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/artifacts?name=$label&per_page=5" \
|
|
| jq -r '[.artifacts[] | select(.expired|not)] | length' || echo 0)
|
|
if (( found > 0 )); then reused+=("$label"); else kept+=("$entry"); fi
|
|
done < <(jq -c '.include[]' <<<"$matrix")
|
|
matrix=$(printf '%s\n' "${kept[@]}" | jq -sc '{include: .}')
|
|
if (( ${#reused[@]} )); then
|
|
printf '==> already built, reusing the artifact: %s\n' "${reused[@]}"
|
|
{ echo "Reused existing build artifacts (${#reused[@]}):"; printf -- '- %s\n' "${reused[@]}"; } >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
fi
|
|
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
|
echo "planned=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT"
|
|
jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix"
|
|
# A PR whose diff against its base is empty changes nothing: its
|
|
# content already landed some other way (a sync PR beat it, or a
|
|
# merge from master swallowed it). Merging it would record a change
|
|
# that isn't one. Flag it so `result` fails rather than passes.
|
|
if [[ "${{ github.event_name }}" == pull_request ]]; then
|
|
total=$(git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" | wc -l)
|
|
echo "empty=$([[ $total -eq 0 ]] && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
|
echo "files changed vs base: $total"
|
|
else
|
|
echo "empty=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
- id: gate
|
|
env:
|
|
STATUS: ${{ github.event_name == 'workflow_dispatch' && 'dispatch' || steps.vouch.outputs.status }}
|
|
AUTHOR: ${{ github.event.pull_request.user.login }}
|
|
APPROVED: ${{ steps.approval.outputs.approved || 'false' }}
|
|
PLANNED: ${{ steps.list.outputs.planned }}
|
|
run: |
|
|
case "$STATUS" in
|
|
bot|collaborator|vouched|dispatch) trusted=true ;;
|
|
# A denouncement is absolute: the label cannot override it.
|
|
denounced) trusted=false ;;
|
|
unknown) trusted=$APPROVED ;;
|
|
*) trusted=false ;;
|
|
esac
|
|
echo "trusted=$trusted" >> "$GITHUB_OUTPUT"
|
|
if [[ $trusted == true ]]; then
|
|
echo "count=$PLANNED" >> "$GITHUB_OUTPUT"
|
|
echo "Author $AUTHOR is trusted ($STATUS); building $PLANNED package(s)."
|
|
else
|
|
echo "count=0" >> "$GITHUB_OUTPUT"
|
|
echo "::warning::Author $AUTHOR is not trusted ($STATUS). $PLANNED package build(s) planned but not run."
|
|
if [[ $STATUS == denounced ]]; then
|
|
echo "::warning::The author is denounced in .github/VOUCHED.td; the build-approved label does not apply."
|
|
else
|
|
echo "::warning::A maintainer can add the author to .github/VOUCHED.td, or apply the 'build-approved' label to build this PR."
|
|
fi
|
|
fi
|
|
|
|
build:
|
|
needs: changes
|
|
if: needs.changes.outputs.count != '0'
|
|
# The droplets are x86, so aarch64 there runs under QEMU: omarchy-mac-boot
|
|
# took 2h47m of the 180 minutes, most of it in check().
|
|
# GitHub's arm64 runners (4 vCPU, 16 GB; ~100 GB disk free in our pilots)
|
|
# build it natively in 11 minutes, and linux-aurora in 30 (72 under QEMU).
|
|
runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || fromJSON('["self-hosted","omarchy-builder"]') }}
|
|
timeout-minutes: 180
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJson(needs.changes.outputs.matrix) }}
|
|
steps:
|
|
# Tooling from base: everything that executes on this runner's host
|
|
# (bin/, helpers/, build/) comes from the base branch. Only the PR's
|
|
# package directories are overlaid. A PR can therefore change what
|
|
# gets built, never how the runner builds it. A PR that changes both
|
|
# tooling and a package builds the package with the OLD tooling; land
|
|
# the tooling first. workflow_dispatch has no PR and runs as checked out.
|
|
# The base branch tip, not the event's base.sha: that sha is a snapshot
|
|
# taken at the PR's last push, so a tooling fix on master would never
|
|
# reach an open PR until someone pushed to it (seen on the daily sync
|
|
# PR after the artifact packing fix landed).
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.event.pull_request.base.ref || github.sha }}
|
|
persist-credentials: false
|
|
- name: Overlay the PR's package directories onto base tooling
|
|
if: github.event_name == 'pull_request'
|
|
run: |
|
|
set -euo pipefail
|
|
git fetch --no-tags --depth=1 origin "${{ github.event.pull_request.head.sha }}"
|
|
git checkout "${{ github.event.pull_request.head.sha }}" -- pkgbuilds/
|
|
echo "tooling: $(git rev-parse --short HEAD) (base) packages: ${{ github.event.pull_request.head.sha }} (PR head)"
|
|
# A preview only. `head` exits after ten lines and, under pipefail,
|
|
# git's SIGPIPE (141) failed the step for any PR far enough behind
|
|
# master to differ in more files; sed reads the whole stream.
|
|
git status --short | sed -n '1,10p'
|
|
# A PR can change a package's directory without changing its version:
|
|
# an upstream hook, its tests, a README. edge already holds that
|
|
# version, so the planner builds nothing and there is nothing to pack.
|
|
# The same check publish.yml's rebuild job makes; without it the pack
|
|
# step failed on the empty output and the required result went red.
|
|
- name: Build ${{ matrix.package }} (${{ matrix.arch }}, ships to ${{ matrix.channels }})
|
|
id: build
|
|
env:
|
|
CONTAINER_ENGINE: docker
|
|
run: |
|
|
set -euo pipefail
|
|
plan=$(bin/build --dry-run --mirror edge --arch "${{ matrix.arch }}" --package "${{ matrix.package }}" 2>&1 | grep -E '^==> Plan complete' || true)
|
|
if [[ -n "$plan" && -z "$(sed -E 's/.*would build: *//' <<<"$plan" | tr -d '[:space:]')" ]]; then
|
|
echo "==> ${{ matrix.package }} (${{ matrix.arch }}): edge already holds this version, nothing to build"
|
|
echo "built=false" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
fi
|
|
bin/build --mirror edge --arch "${{ matrix.arch }}" --package "${{ matrix.package }}"
|
|
echo "built=true" >> "$GITHUB_OUTPUT"
|
|
# makepkg's check() leaves meson's per-test output in the build tree,
|
|
# never on stdout, so a failing test shows only a summary line in this
|
|
# job log. bin/build bind-mounts $SRC_DIR at /src, so those logs outlive
|
|
# the container. Without this upload an arch-specific test failure
|
|
# cannot be diagnosed from CI at all (seen on owe 0.2.7, aarch64).
|
|
- name: Upload test logs
|
|
if: always() && steps.build.outcome == 'failure'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: test-logs-${{ matrix.package }}-${{ matrix.arch }}
|
|
path: src/**/meson-logs/
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
# The artifact label carries the package directory's git tree hash so
|
|
# the publish step can find the build for exactly the tree that merged.
|
|
# The package file inside keeps makepkg's standard name untouched.
|
|
# The artifact label uses the PR head's tree for this package: that is
|
|
# the tree that merges, and what publish looks up.
|
|
- name: Tree hash
|
|
id: tree
|
|
run: echo "hash=$(git rev-parse "${{ github.event.pull_request.head.sha || github.sha }}:pkgbuilds/${{ matrix.package }}")" >> "$GITHUB_OUTPUT"
|
|
# The upload action rejects a path containing ':', which is how makepkg
|
|
# names a package with an epoch. The files ride inside packages.tar
|
|
# (helpers/artifact-helpers.sh); publish.yml unpacks it. Only a
|
|
# successful build uploads: the artifact's existence is what lets the
|
|
# planner above and publish.yml skip rebuilding this exact tree.
|
|
- name: Pack artifact
|
|
if: steps.build.outputs.built == 'true'
|
|
id: pack
|
|
run: |
|
|
source helpers/artifact-helpers.sh
|
|
pack_packages build-output/edge/${{ matrix.arch }} packages.tar
|
|
tar -tvf packages.tar
|
|
- name: Upload artifact
|
|
if: steps.build.outputs.built == 'true'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ${{ matrix.package }}-${{ matrix.arch }}-${{ steps.tree.outputs.hash }}
|
|
path: packages.tar
|
|
if-no-files-found: error
|
|
retention-days: 7
|
|
|
|
# `result` is required by branch protection. An unvouched author awaiting
|
|
# approval gets a differently named informational check, leaving `result`
|
|
# unreported (pending). Skipping or passing a job named `result` would count
|
|
# as satisfying the requirement even though no build was authorized.
|
|
# Actual planning/build failures and denouncements still report `result`.
|
|
result:
|
|
name: ${{ needs.changes.result == 'success' && needs.changes.outputs.trusted == 'false' && needs.changes.outputs.vouch_status == 'unknown' && needs.changes.outputs.empty == 'false' && 'Awaiting build approval' || 'result' }}
|
|
needs: [changes, build]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- run: |
|
|
echo "trusted=${{ needs.changes.outputs.trusted }} build=${{ needs.build.result }}"
|
|
if [[ "${{ needs.changes.result }}" != "success" ]]; then
|
|
echo "::error::Build planning or the trust check failed. See the changes job."
|
|
exit 1
|
|
fi
|
|
# Nothing to merge: the PR's diff against its base is empty. Its
|
|
# change already landed elsewhere. Close it rather than merge it.
|
|
if [[ "${{ needs.changes.outputs.empty }}" == "true" ]]; then
|
|
echo "::error::This PR changes no files relative to its base. Its content is already on the target branch; close it instead of merging."
|
|
exit 1
|
|
fi
|
|
if [[ "${{ needs.changes.outputs.trusted }}" == "false" && "${{ needs.changes.outputs.vouch_status }}" == "unknown" && "${{ needs.changes.outputs.empty }}" == "false" ]]; then
|
|
echo "::notice::Awaiting maintainer build approval. Apply 'build-approved' to this PR or vouch for the author in .github/VOUCHED.td."
|
|
echo "Package builds are waiting for maintainer approval. Apply **build-approved** to this PR to start them. The required **result** check remains pending." >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
if [[ "${{ needs.changes.outputs.trusted }}" != "true" ]]; then
|
|
echo "::error::Builds are blocked: the author is denounced or the trust result is invalid. The build-approved label cannot override this."
|
|
exit 1
|
|
fi
|
|
[[ "${{ needs.build.result }}" == "success" || "${{ needs.build.result }}" == "skipped" ]]
|