diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0dde694..bbc3aaf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,11 +24,10 @@ on: description: "Space-separated package directories to publish from master" required: true -# Merges serialize. Two publishes into one channel at once would race on -# the database; queued is fine, cancelled is not. -concurrency: - group: publish - cancel-in-progress: false +# Only the publish job serializes (see its concurrency group): two publishes +# into one channel at once would race on the database. Builds run outside the +# lock, so a merge waits behind another merge's signing and upload, seconds, +# never behind its kernel build. jobs: changes: @@ -58,11 +57,10 @@ jobs: echo "matrix=$matrix" >> "$GITHUB_OUTPUT" echo "count=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT" jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix" - # Reuse or rebuild, decided per entry and said out loud. An aarch64 - # tree with no build artifact (PR artifacts last 7 days; a dispatch - # may name any package) goes to the rebuild job, which builds it - # natively on GitHub's arm64 runner. x86_64 builds inside the - # publish job on the droplet, as before. + # Reuse or rebuild, decided per entry and said out loud. A tree with + # no build artifact (PR artifacts last 7 days; a dispatch may name + # any package) goes to the rebuild job: aarch64 natively on GitHub's + # arm64 runner, x86_64 on a builder droplet, one runner per entry. rebuild=() echo "| package | arch | tree | decision |" >> "$GITHUB_STEP_SUMMARY" echo "| --- | --- | --- | --- |" >> "$GITHUB_STEP_SUMMARY" @@ -77,9 +75,10 @@ jobs: decision="reuse the build artifact ($found)" elif [[ $arch == aarch64 ]]; then decision="no build artifact: rebuild natively on ubuntu-24.04-arm" - rebuild+=("$entry") + rebuild+=("$(jq -c '. + {runner: "[\"ubuntu-24.04-arm\"]"}' <<<"$entry")") else - decision="no build artifact: build in the publish job on the self-hosted builder" + decision="no build artifact: rebuild on a builder droplet" + rebuild+=("$(jq -c '. + {runner: "[\"self-hosted\",\"omarchy-builder\"]"}' <<<"$entry")") fi echo "==> $label: $decision" echo "| $package | $arch | ${hash:0:12} | $decision |" >> "$GITHUB_STEP_SUMMARY" @@ -87,16 +86,19 @@ jobs: echo "rebuild=$(printf '%s\n' "${rebuild[@]}" | jq -sc '{include: .}')" >> "$GITHUB_OUTPUT" echo "rebuild_count=${#rebuild[@]}" >> "$GITHUB_OUTPUT" - # The aarch64 half of "build it now when there is none". It builds exactly - # as build-pr.yml's aarch64 path does (same runner, same builder image, - # same bin/build call) and uploads under the same label, so the publish - # job collects this run's artifact the way it collects a PR's. No secret - # reaches this runner; signing and upload stay on the self-hosted builder. + # "Build it now when there is none". Each entry builds exactly as + # build-pr.yml builds it (same runner kind, same builder image, same + # bin/build call) and uploads under the same label, so the publish job + # collects this run's artifact the way it collects a PR's. No secret + # reaches these runners, and they hold no lock: entries build in parallel, + # and other merges publish while they do. rebuild: needs: changes if: needs.changes.outputs.rebuild_count != '0' - runs-on: ubuntu-24.04-arm - timeout-minutes: 180 + runs-on: ${{ fromJSON(matrix.runner) }} + # The droplets are x86, so aarch64 never builds there. omarchy-mac-boot + # under QEMU took 2h47m; native arm64 and x86 kernels fit easily. + timeout-minutes: 240 permissions: contents: read strategy: @@ -109,7 +111,7 @@ jobs: # The same check the publish job makes before building: a re-run for a # package the channel already holds at master's version builds # nothing, and uploads nothing that could shadow the published file. - - name: Build ${{ matrix.package }} (${{ matrix.arch }}, native) + - name: Build ${{ matrix.package }} (${{ matrix.arch }}) id: build env: CONTAINER_ENGINE: docker @@ -140,23 +142,26 @@ jobs: if-no-files-found: error retention-days: 7 - # One job for the whole merge. It collects every PR artifact for the - # merged tree (building only what has none; aarch64 comes from the - # rebuild job above), then walks each channel and + # One job for the whole merge. It collects every artifact for the merged + # tree (PR builds, or the rebuild job above), then walks each channel and # architecture slot exactly once: pull that database, add every package # that belongs in it, upload. Six slots, six round trips, however many # packages the merge carried. One process is the only writer, so there - # is no race between packages; the run-level concurrency group above - # keeps one merge from overlapping the next. + # is no race between packages; the concurrency group keeps one merge's + # publish from overlapping the next. It builds nothing, so it runs on a + # hosted runner in about a minute instead of waiting for a droplet. # It waits for the rebuild job and runs whatever that job's result: a # failed rebuild leaves its package without an artifact, and the collect # step below records that and stops before any publish. publish: needs: [changes, rebuild] if: ${{ !cancelled() && needs.changes.result == 'success' && needs.changes.outputs.count != '0' }} - runs-on: [self-hosted, omarchy-builder] + runs-on: ubuntu-latest environment: publish - timeout-minutes: 240 + timeout-minutes: 30 + concurrency: + group: publish + cancel-in-progress: false steps: - uses: actions/checkout@v4 with: @@ -172,8 +177,8 @@ jobs: EOF_MATRIX cat plan.txt - # Fetch each package's PR artifact into build-output/edge//, or - # build it when no artifact exists for exactly this tree. An artifact + # Fetch each package's artifact into build-output/edge//: the PR's, + # or the rebuild job's when the PR's had expired. An artifact # carries its package files inside packages.tar (see build-pr.yml and # helpers/artifact-helpers.sh: the upload action rejects the colon in # an epoch filename). @@ -199,8 +204,8 @@ jobs: mkdir -p "build-output/edge/$arch" if [[ -n "$found" ]]; then if [[ $from_run == "${{ github.run_id }}" ]]; then - kind=native-rebuild - echo "==> $label: artifact from this run's native $arch rebuild" + kind=rebuild + echo "==> $label: artifact from this run's $arch rebuild" else kind=pr-artifact echo "==> $label: reusing the build artifact from run $from_run" @@ -225,20 +230,11 @@ jobs: jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"already-published"}' >> sources.jsonl continue fi - # aarch64 never builds here: this droplet is x86 and would - # emulate it. No artifact means the native rebuild failed (see - # the rebuild job), or an artifact expired between planning - # and now (re-run all jobs). - if [[ $arch == aarch64 ]]; then - echo "::error::$label: no artifact from the native rebuild; not building aarch64 under emulation" - jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"native-build-failed"}' >> sources.jsonl; failed=1; break - fi - echo "==> $label: no artifact for this tree, building" - if OMARCHY_KEEP_BUILD_WORKSPACE=1 bin/build --mirror edge --arch "$arch" --package "$package"; then - jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"built"}' >> sources.jsonl - else - jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"build-failed"}' >> sources.jsonl; failed=1; break - fi + # Nothing builds here. No artifact means the rebuild failed (see + # the rebuild job), or an artifact expired between planning and + # now (re-run all jobs). + echo "::error::$label: no artifact from the rebuild job" + jq -nc --arg p "$package" --arg a "$arch" '{package:$p, arch:$a, source:"build-failed"}' >> sources.jsonl; failed=1; break fi done < plan.txt ls -1 build-output/edge/*/*.pkg.tar.zst 2>/dev/null || true @@ -280,8 +276,8 @@ jobs: cat publish-record.json exit 0 fi - # A builder droplet starts with no images, so building this one here - # cost every publish about 100 s (and 20 s more to start a container + # A fresh runner has no images, and building this one here cost + # every publish about 100 s (and 20 s more to start a container # from it) for the 14 s of signing and upload it is needed for. # builder-images.yml already publishes the tested image for exactly # these build inputs under their key; pull that. Build only when no @@ -392,7 +388,7 @@ jobs: run: | jq -r --arg outcome "${{ needs.publish.result }}" ' def pkgs: [.packages[] | select(test("-debug-")|not)] | join(", "); - def src: if .source=="pr-artifact" then "PR artifact" elif .source=="native-rebuild" then "rebuilt natively on arm64" elif .source=="built" then "built here" elif .source=="already-published" then "already published, skipped" else "**" + .source + "**" end; + def src: if .source=="pr-artifact" then "PR artifact" elif .source=="rebuild" or .source=="native-rebuild" then "rebuilt at merge" elif .source=="built" then "built here" elif .source=="already-published" then "already published, skipped" else "**" + .source + "**" end; "### Publish " + (if $outcome=="success" then "succeeded" else "FAILED" end) + " → **" + .target + "**" + (if .target!="live" then " (proof prefix, not live)" else "" end), "", diff --git a/ci/README.md b/ci/README.md index d0d0739..489862b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -80,8 +80,11 @@ changing them, on the box: packages then signatures then the db. - aarch64 under QEMU with credential-preserving binfmt. PR builds now run aarch64 natively on `ubuntu-24.04-arm` (QEMU was up to ~15x slower). When a - merged aarch64 tree has no artifact, publish.yml rebuilds it there too, in its - own job, and signs and uploads it on the droplet like a PR artifact. + merged tree has no artifact, publish.yml rebuilds it in its own job, aarch64 + there and x86_64 on a droplet, outside the publish lock. +- Publish itself builds nothing: it signs and uploads on `ubuntu-latest` in the + tested builder image pulled from GHCR, and only that job holds the `publish` + concurrency group, so a merge waits for seconds of signing, not for builds. - Vouch gate: collaborators, `.github/VOUCHED.td`, or the `build-approved` label; denounced authors cannot be overridden by the label. - Tests run on PRs only; `result`, `self-tests`, `build-isolation` are the