diff --git a/.github/scripts/approve-sync-push.cjs b/.github/scripts/approve-sync-push.cjs deleted file mode 100644 index ecb94db8..00000000 --- a/.github/scripts/approve-sync-push.cjs +++ /dev/null @@ -1,34 +0,0 @@ -const approvePrWorkflows = require('./approve-pr-workflows.cjs'); - -const BOT = 'github-actions[bot]'; - -// A sync workflow pushes its branch with GITHUB_TOKEN. GitHub holds the -// resulting pull_request runs for approval and, unlike a person's push, -// creates no pull_request_target run, so approve-pr.yml never sees it. The -// sync workflow therefore releases the runs for the commit it just pushed, -// under the same rule approve-pr.yml applies: only while build-approved is -// on the PR. The sync applies that label itself, so its pushes build without -// a maintainer. It acts only on its own bot-authored, same-repository PR for -// the branch and commit it pushed. -module.exports = async function approveSyncPush({ github, context, core, - number, branch, headSha, since, approve = approvePrWorkflows, ...options }) { - if (!Number.isInteger(number) || !branch || !headSha || !since) { - throw new Error('Missing sync PR number, branch, head SHA or push time.'); - } - const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: number }); - const repository = `${context.repo.owner}/${context.repo.repo}`; - if (pr.user?.login !== BOT || pr.head.repo?.full_name !== repository || - pr.base.repo?.full_name !== repository || pr.head.ref !== branch) { - throw new Error(`PR #${number} is not ${BOT}'s ${branch} PR in ${repository}; refusing to approve.`); - } - if (pr.state !== 'open' || pr.head.sha !== headSha) { - core.info(`PR #${number} is closed or has moved past ${headSha}; nothing to approve.`); - return; - } - if (!pr.labels.some(label => label.name === 'build-approved')) { - core.info(`PR #${number} has no build-approved label; its runs stay held.`); - return; - } - await approve({ github, context, core, vouchStatus: 'bot', pullRequest: pr, - action: 'synchronize', since, ...options }); -}; diff --git a/.github/scripts/auto-merge-pr.cjs b/.github/scripts/auto-merge-pr.cjs index 982bf995..c1544db3 100644 --- a/.github/scripts/auto-merge-pr.cjs +++ b/.github/scripts/auto-merge-pr.cjs @@ -14,9 +14,6 @@ // test.yml runs on PRs only; an existing test may also run on master // (builder-images.yml runs tests/build-isolation.sh with packages: write), // so editing one still needs a maintainer. -// - Not the upstream sync. It labels its own PR build-approved to release -// GitHub's hold on its pushes, which is no one's approval; it stays on the -// reviewed lane. const PACKAGES = 'pkgbuilds/'; const TESTS = 'tests/'; const TEST_WORKFLOW = '.github/workflows/test.yml'; @@ -38,7 +35,6 @@ function packageChange(file) { if (file.filename === TEST_WORKFLOW) return file.status === 'modified' && onlyRunsTests(file.patch); return false; } -const REVIEWED_BRANCHES = /^auto\/sync-upstream(\/|$)/; function decide({ pr, files, vouchStatus, repository }) { if (pr.state !== 'open') return { enable: false, reason: 'PR is not open' }; @@ -55,10 +51,6 @@ function decide({ pr, files, vouchStatus, repository }) { return { enable: false, reason: `author not trusted to build (${vouchStatus || 'missing'}${labelled ? ', labelled' : ''})` }; } - if (pr.head.repo?.full_name === repository && REVIEWED_BRANCHES.test(pr.head.ref)) { - return { enable: false, reason: `${pr.head.ref} stays on the reviewed lane` }; - } - if (!files.length) return { enable: false, reason: 'PR changes no files' }; const outside = files.filter(file => !packageChange(file)); if (outside.length) { diff --git a/.github/scripts/package-pr-groups.sh b/.github/scripts/package-pr-groups.sh new file mode 100755 index 00000000..5c977335 --- /dev/null +++ b/.github/scripts/package-pr-groups.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Usage: package-pr-groups.sh BRANCH_PREFIX PATCH_OUT +# +# Run from the repository root after a sync has rewritten recipes in the +# working tree. Writes those changes to PATCH_OUT and prints, as one JSON +# line, the PRs they split into: [{"branch": ..., "packages": "a b"}, ...]. +# +# One PR per package, so a package whose build fails holds back only itself. +# Packages pinned from the same upstream branch (omarchy-dev and +# omarchy-settings-dev) share a PR: sync-upstream moves them in lockstep, and +# merging one without the other would ship the pair from different commits. +# Branch names come from sync-pr-branch.sh, so a scheduled run and a dispatch +# naming the same packages update the same PR. +set -euo pipefail + +prefix=${1:?branch prefix required} +patch=${2:?patch path required} +scripts=$(dirname "$0") + +git diff --binary -- pkgbuilds > "$patch" + +declare -A members=() +while IFS= read -r package; do + key=$(jq -r '(.upstream.watch? | objects | select(has("git_branch")) | "\(.git_branch)#\(.branch)") // empty' \ + "pkgbuilds/$package/.omarchy/package.json" 2>/dev/null || true) + members["${key:-$package}"]+="$package " +done < <(git diff --name-only -- pkgbuilds | cut -d/ -f2 | sort -u) + +groups='[]' +for key in "${!members[@]}"; do + read -r -a packages <<<"${members[$key]}" + branch=$("$scripts/sync-pr-branch.sh" "$prefix" "${packages[@]}" | sed -n 's/^branch=//p') + groups=$(jq -c --arg branch "$branch" --arg packages "${packages[*]}" \ + '. + [{branch: $branch, packages: $packages}]' <<<"$groups") +done +jq -c 'sort_by(.branch)' <<<"$groups" diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 611d373d..ed0a70b1 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -21,7 +21,7 @@ on: required: true # A new push normally cancels the PR's in-flight build. The bots' branches -# (auto/sync-* and auto/track-branches) are the exception: they are +# (auto/sync-* and auto/track-branches-*) are the exception: they are # force-pushed with fresh upstream releases several times a day, which kept # cancelling multi-hour aarch64 builds before they could finish. There the # newest run waits instead (GitHub keeps at most one pending run per group, @@ -31,7 +31,7 @@ on: # runs at once; cancelling the first left a red `result` on the PR (#901). concurrency: group: build-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ !(github.event.pull_request.head.repo.full_name == github.repository && (startsWith(github.head_ref, 'auto/sync-') || github.head_ref == 'auto/track-branches')) }} + cancel-in-progress: ${{ !(github.event.pull_request.head.repo.full_name == github.repository && (startsWith(github.head_ref, 'auto/sync-') || startsWith(github.head_ref, 'auto/track-branches'))) }} jobs: # Builds cost real machines, so they run only for trusted authors: diff --git a/.github/workflows/package-prs.yml b/.github/workflows/package-prs.yml new file mode 100644 index 00000000..1b8c64e2 --- /dev/null +++ b/.github/workflows/package-prs.yml @@ -0,0 +1,134 @@ +name: Open package PRs + +# Shared by the three bot lanes (sync-upstream.yml, sync-rebuilds.yml and +# track-branches.yml). The caller's sync job rewrites recipes, groups the +# changes with .github/scripts/package-pr-groups.sh and uploads them as the +# `package-pins` artifact. This opens or updates one PR per group and arms +# auto-merge, so every green package lands on its own and only the failures +# stay open. A single batch PR let one red package hold back every other one +# (#788, #837). +# +# Everything goes through the PAT in PKGS_BOT_TOKEN: its pushes build without +# an approval hold, and a merge made with it starts publish.yml. A merge made +# with the built-in GITHUB_TOKEN would not publish. + +on: + workflow_call: + inputs: + groups: + description: 'JSON from package-pr-groups.sh: [{"branch", "packages"}]' + required: true + type: string + prefix: + description: 'Branch prefix the caller owns, e.g. auto/sync-upstream' + required: true + type: string + title: + description: 'PR title; ": , ..." is appended' + required: true + type: string + body: + description: 'PR body' + required: true + type: string + secrets: + PKGS_BOT_TOKEN: + required: true + +permissions: + contents: read + +jobs: + pr: + if: ${{ inputs.groups != '[]' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 4 + matrix: + group: ${{ fromJSON(inputs.groups) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Fetch the pins + uses: actions/download-artifact@v4 + with: + name: package-pins + path: ${{ runner.temp }} + + # The PR title names what moved, so the merged history reads like a + # changelog: "Sync upstream releases: zed 1.23.2-1". + - name: Apply this PR's packages + id: apply + env: + PACKAGES: ${{ matrix.group.packages }} + TITLE: ${{ inputs.title }} + run: | + read -r -a packages <<<"$PACKAGES" + git apply $(printf -- '--include=pkgbuilds/%s/* ' "${packages[@]}") "$RUNNER_TEMP/pins.patch" + git status --porcelain + field() { sed -n "s/^$2=//p" "pkgbuilds/$1/PKGBUILD" | head -1 | tr -d "\"'"; } + versions=$(for package in "${packages[@]}"; do + printf '%s %s-%s, ' "$package" "$(field "$package" pkgver)" "$(field "$package" pkgrel)" + done) + title="$TITLE: ${versions%, }" + echo "title=$title" >> "$GITHUB_OUTPUT" + echo "- $title" >> "$GITHUB_STEP_SUMMARY" + + - name: Open or update the PR + id: pr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ secrets.PKGS_BOT_TOKEN }} + commit-message: ${{ steps.apply.outputs.title }} + title: ${{ steps.apply.outputs.title }} + body: ${{ inputs.body }} + branch: ${{ matrix.group.branch }} + delete-branch: true + labels: automated + + # Auto-merge, not a direct merge: branch protection still has to see + # `result`, `self-tests` and `build-isolation` green. auto-merge-pr.yml + # may arm the same PR at the same moment; either one arming it is the + # outcome wanted. + - name: Enable auto-merge + if: ${{ steps.pr.outputs.pull-request-number != '' }} + env: + GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + PR: ${{ steps.pr.outputs.pull-request-number }} + run: | + armed() { [[ "$(gh pr view "$PR" -R "$GITHUB_REPOSITORY" --json autoMergeRequest --jq '.autoMergeRequest != null')" == true ]]; } + if armed; then + echo "auto-merge already enabled on #$PR" + exit 0 + fi + gh pr merge --auto --merge "$PR" -R "$GITHUB_REPOSITORY" || armed + + # A PR on this lane's prefix that the run did not refresh, and that + # conflicts with master, is stale: master moved its packages on without it. + # Close it rather than leave a red PR behind. Not being refreshed is not + # enough on its own, since a feed that failed this run (one of dozens timing + # out is routine) leaves a still-valid PR unrefreshed too. The lane's old + # batch PR, on the bare prefix, is retired outright. + close-superseded: + runs-on: ubuntu-latest + steps: + - name: Close superseded PRs + env: + GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + GROUPS_JSON: ${{ inputs.groups }} + PREFIX: ${{ inputs.prefix }} + run: | + gh pr list -R "$GITHUB_REPOSITORY" --state open --limit 500 --json number,headRefName,mergeable \ + --jq '.[] | select(.headRefName == env.PREFIX or (.headRefName | startswith(env.PREFIX + "-"))) | "\(.number) \(.headRefName) \(.mergeable)"' | + while read -r number branch mergeable; do + jq -e --arg branch "$branch" 'any(.[]; .branch == $branch)' <<<"$GROUPS_JSON" >/dev/null && continue + [[ $branch == "$PREFIX" || $mergeable == CONFLICTING ]] || continue + echo "closing #$number ($branch): superseded" + gh pr close "$number" -R "$GITHUB_REPOSITORY" --delete-branch \ + --comment "Superseded: master has moved these packages on, or per-package PRs carry them now." + done diff --git a/.github/workflows/sync-rebuilds.yml b/.github/workflows/sync-rebuilds.yml index f57b0ac9..9bbda0c3 100644 --- a/.github/workflows/sync-rebuilds.yml +++ b/.github/workflows/sync-rebuilds.yml @@ -1,9 +1,11 @@ name: Sync Rebuild Triggers -# Rebuilds ride the unattended lane, like track-branches.yml: the pkgrel bump -# PR builds on the droplets, auto-merge lands it once `result`, `self-tests` -# and `build-isolation` are green, and the merge publishes. A rebuild that -# fails stays an unmerged red PR for a maintainer. +# Rebuilds ride the unattended lane, like track-branches.yml: each package's +# pkgrel bump gets its own PR on auto/sync-rebuilds-, opened by +# package-prs.yml. It builds on the droplets, auto-merge lands it once +# `result`, `self-tests` and `build-isolation` are green, and the merge +# publishes. A rebuild that fails stays an unmerged red PR for a maintainer +# without holding back any other package. # # The PR and auto-merge use the PAT in PKGS_BOT_TOKEN, because a merge made # with the built-in GITHUB_TOKEN does not start publish.yml, and its pushes @@ -20,12 +22,19 @@ on: required: false default: '' +# One sync at a time: two runs racing on one auto/sync-rebuilds-* branch +# would each force-push their own bump over the other's. +concurrency: + group: sync-rebuilds + cancel-in-progress: false + jobs: sync: runs-on: ubuntu-latest permissions: - contents: write - pull-requests: write + contents: read + outputs: + groups: ${{ steps.groups.outputs.groups }} steps: - name: Require the bot token @@ -42,23 +51,13 @@ jobs: with: persist-credentials: false - # A scoped dispatch regenerates only the named packages. Pushed to the - # shared branch, that would replace every other pending update in its - # PR, so it gets a branch and PR of its own. - - name: Choose the PR branch - id: branch - env: - PACKAGES: ${{ github.event.inputs.packages }} - run: | - read -r -a package_args <<< "${PACKAGES:-}" - .github/scripts/sync-pr-branch.sh auto/sync-rebuilds "${package_args[@]}" | tee -a "$GITHUB_OUTPUT" - # Runs in an Arch container against the mirror the x86_64 builder itself # uses, because the question being asked is what that builder will link # against and a different mirror can be hours ahead of it. Recording a # version the build never saw is the one failure this command must not # have: nothing re-fires once the record matches. - name: Bump pkgrel for packages whose dependencies moved + id: sync run: | docker run --rm \ -e PACKAGES="$PACKAGES" \ @@ -88,56 +87,53 @@ jobs: env: PACKAGES: ${{ github.event.inputs.packages }} - - name: Check for changes - id: changes + - name: Group the bumps into PRs + if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }} + id: groups run: | - if [ -z "$(git status --porcelain)" ]; then - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - fi + groups=$(.github/scripts/package-pr-groups.sh auto/sync-rebuilds "$RUNNER_TEMP/pins.patch") + echo "groups=$groups" >> "$GITHUB_OUTPUT" - - name: Create Pull Request - if: steps.changes.outputs.has_changes == 'true' - id: cpr - uses: peter-evans/create-pull-request@v7 + - name: Hand the bumps to the PR jobs + if: ${{ !cancelled() && steps.groups.outputs.groups != '' && steps.groups.outputs.groups != '[]' }} + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.PKGS_BOT_TOKEN }} - commit-message: 'chore: rebuild against updated dependencies' - title: "chore: rebuild against updated dependencies${{ steps.branch.outputs.scope && format(' ({0})', steps.branch.outputs.scope) || '' }}" - body: | - Automated pkgrel bump for packages that link against a dependency - which has moved in the official repositories. + name: package-pins + path: ${{ runner.temp }}/pins.patch + retention-days: 1 - Each package names those dependencies in `rebuild_on` and carries the - versions its current pkgrel was bumped for in `rebuilt_against`. The - bump is what makes the rebuilt package an upgrade pacman will offer; - without it the build produces the version already published and no - one receives it. + prs: + needs: sync + if: ${{ !cancelled() && needs.sync.outputs.groups != '' }} + permissions: + contents: read + uses: ./.github/workflows/package-prs.yml + with: + groups: ${{ needs.sync.outputs.groups }} + prefix: auto/sync-rebuilds + title: Rebuild against updated dependencies + body: | + Automated pkgrel bump for a package that links against a dependency + which has moved in the official repositories. - This PR auto-merges once the build checks pass. A failing rebuild - leaves it open for a maintainer. - branch: ${{ steps.branch.outputs.branch }} - delete-branch: true - labels: automated + Each package names those dependencies in `rebuild_on` and carries the + versions its current pkgrel was bumped for in `rebuilt_against`. The + bump is what makes the rebuilt package an upgrade pacman will offer; + without it the build produces the version already published and no + one receives it. - # Auto-merge, not a direct merge: branch protection still has to see - # the build checks green before the rebuild lands. - - name: Enable auto-merge - if: steps.cpr.outputs.pull-request-number != '' - env: - GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} - PR: ${{ steps.cpr.outputs.pull-request-number }} - run: | - # Idempotent across re-runs of an updated PR: enabling twice errors. - if [[ "$(gh pr view "$PR" -R "${{ github.repository }}" --json autoMergeRequest --jq '.autoMergeRequest != null')" == true ]]; then - echo "auto-merge already enabled on #$PR" - exit 0 - fi - gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" + This PR auto-merges once the build checks pass. A failing rebuild + leaves it open for a maintainer without holding back any other package. + secrets: + PKGS_BOT_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + notify: + needs: [sync, prs] + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: - name: Notify Basecamp on failure - if: failure() && env.BASECAMP_CHATBOT_URL != '' + if: ${{ env.BASECAMP_CHATBOT_URL != '' }} env: BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} run: | diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 7213624d..f7fd7d36 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -1,5 +1,17 @@ name: Sync Upstream Releases +# Bumps every package whose upstream release feed has moved, except the +# "auto_merge": true packages that track-branches.yml checks hourly. Each +# package gets its own PR on auto/sync-upstream-, opened by +# package-prs.yml: the PR builds on the droplets, auto-merge lands it when +# `result` is green, and the merge publishes. A package that fails to build +# stays an open red PR for a maintainer without holding back any other; the +# next run updates it if upstream moves again. `min_release_age` still +# quarantines fresh releases before they are proposed. +# +# The PRs and auto-merge use the PAT in PKGS_BOT_TOKEN, so their pushes +# build without an approval hold and their merges start publish.yml. + on: schedule: # Every 6 hours, off the hour to dodge the scheduling backlog at :00 @@ -11,36 +23,35 @@ on: required: false default: '' +# One sync at a time: two runs racing on one auto/sync-upstream-* branch +# would each force-push their own update over the other's. +concurrency: + group: sync-upstream + cancel-in-progress: false + jobs: sync: runs-on: ubuntu-latest permissions: - contents: write - pull-requests: write + contents: read outputs: - branch: ${{ steps.branch.outputs.branch }} - pushed_at: ${{ steps.pushed.outputs.at }} - number: ${{ steps.cpr.outputs.pull-request-number }} - operation: ${{ steps.cpr.outputs.pull-request-operation }} - head_sha: ${{ steps.cpr.outputs.pull-request-head-sha }} + groups: ${{ steps.groups.outputs.groups }} steps: + - name: Require the bot token + env: + PKGS_BOT_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + run: | + if [[ -z "$PKGS_BOT_TOKEN" ]]; then + echo "::error::Set PKGS_BOT_TOKEN to a PAT with Contents: write and Pull requests: write on this repository, owned by an account trusted to trigger builds." + exit 1 + fi + - name: Checkout repository uses: actions/checkout@v4 with: persist-credentials: false - # A scoped dispatch regenerates only the named packages. Pushed to the - # shared branch, that would replace every other pending update in its - # PR, so it gets a branch and PR of its own. - - name: Choose the PR branch - id: branch - env: - PACKAGES: ${{ github.event.inputs.packages }} - run: | - read -r -a package_args <<< "${PACKAGES:-}" - .github/scripts/sync-pr-branch.sh auto/sync-upstream "${package_args[@]}" | tee -a "$GITHUB_OUTPUT" - # Runs in an Arch container for vercmp: whether a release is an upgrade has # to be decided by the same comparator pacman will use on users' machines. - name: Update packages from upstream release feeds @@ -64,8 +75,8 @@ jobs: useradd -m -u "$HOST_UID" -g "$HOST_GID" runner chown -R runner:runner /workspace/pkgbuilds - # The reviewed lane only: packages marked auto_merge ride - # track-branches.yml, which merges without a human. + # Packages marked auto_merge ride track-branches.yml instead, + # which checks them hourly. if [[ -n "${PACKAGES:-}" ]]; then read -r -a package_args <<< "$PACKAGES" runuser -u runner -- ./bin/sync-upstream --lane reviewed "${package_args[@]}" @@ -78,50 +89,50 @@ jobs: UPSTREAM_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Failed feeds leave their recipes untouched; completed updates still - # reach review. The failed sync step keeps the workflow red and notifies. - - name: Check for changes + # get their PRs. The failed sync step keeps the workflow red and notifies. + - name: Group the updates into PRs if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }} - id: changes + id: groups run: | - if [ -z "$(git status --porcelain)" ]; then - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - fi + groups=$(.github/scripts/package-pr-groups.sh auto/sync-upstream "$RUNNER_TEMP/pins.patch") + echo "groups=$groups" >> "$GITHUB_OUTPUT" - # Runs created by this push are newer than this; the approve job - # waits for them. A minute's slack absorbs runner clock skew. - - name: Record push time - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: pushed - run: echo "at=$(date -u -d '-1 minute' +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" - - - name: Create Pull Request - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: cpr - uses: peter-evans/create-pull-request@v7 + - name: Hand the updates to the PR jobs + if: ${{ !cancelled() && steps.groups.outputs.groups != '' && steps.groups.outputs.groups != '[]' }} + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: 'chore: sync upstream releases' - title: "chore: sync upstream releases${{ steps.branch.outputs.scope && format(' ({0})', steps.branch.outputs.scope) || '' }}" - body: | - Automated update of packages that track an upstream vendor release - feed rather than the AUR. + name: package-pins + path: ${{ runner.temp }}/pins.patch + retention-days: 1 - Release watches and providers are declared in `.omarchy/package.json`; - exceptional feeds use `.omarchy/upstream.sh`. Failed package updates - are left untouched; check the workflow result for outstanding failures. - branch: ${{ steps.branch.outputs.branch }} - delete-branch: true - # The bot is trusted; build-approved lets the approve job below - # release GitHub's hold on its pushes without a maintainer. - labels: | - automated - build-approved - reviewers: ryanrhughes + prs: + needs: sync + if: ${{ !cancelled() && needs.sync.outputs.groups != '' }} + permissions: + contents: read + uses: ./.github/workflows/package-prs.yml + with: + groups: ${{ needs.sync.outputs.groups }} + prefix: auto/sync-upstream + title: Sync upstream releases + body: | + Automated update of a package that tracks an upstream vendor release + feed rather than the AUR. + Release watches and providers are declared in `.omarchy/package.json`; + exceptional feeds use `.omarchy/upstream.sh`. This PR auto-merges once + the build checks pass. A failing build leaves it open for a maintainer + without holding back any other package. + secrets: + PKGS_BOT_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + + notify: + needs: [sync, prs] + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: - name: Notify Basecamp on failure - if: failure() && env.BASECAMP_CHATBOT_URL != '' + if: ${{ env.BASECAMP_CHATBOT_URL != '' }} env: BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} run: | @@ -131,36 +142,3 @@ jobs: "🔴 Upstream sync failed
View run" \ '{content: $content}')" \ "$BASECAMP_CHATBOT_URL" - - # GitHub holds pull_request runs from a GITHUB_TOKEN push for approval and - # creates no pull_request_target run for it, so approve-pr.yml never sees - # the sync's own pushes. The sync labels its PR build-approved, so release - # the held runs for the commit just pushed, whether it opened the PR or - # updated it. A separate job, so the sync container's token never holds - # actions: write. - approve: - needs: sync - if: ${{ !cancelled() && (needs.sync.outputs.operation == 'created' || needs.sync.outputs.operation == 'updated') }} - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: read - actions: write - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Release held build and test runs - uses: actions/github-script@v7 - env: - NUMBER: ${{ needs.sync.outputs.number }} - BRANCH: ${{ needs.sync.outputs.branch }} - HEAD_SHA: ${{ needs.sync.outputs.head_sha }} - SINCE: ${{ needs.sync.outputs.pushed_at }} - with: - script: | - const approve = require('./.github/scripts/approve-sync-push.cjs'); - const { NUMBER, BRANCH, HEAD_SHA, SINCE } = process.env; - await approve({ github, context, core, number: Number(NUMBER), - branch: BRANCH, headSha: HEAD_SHA, since: SINCE }); diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml index 2894d36e..a80f251c 100644 --- a/.github/workflows/track-branches.yml +++ b/.github/workflows/track-branches.yml @@ -1,15 +1,21 @@ name: Track upstream branches -# The unattended lane. Packages marked "auto_merge": true update without a -# human: some follow a moving upstream branch (omarchy-dev and -# omarchy-settings-dev on quattro, omasnap-git on main) rather than tagged -# releases, so nothing in this repository changes when their source does; -# the rest are release feeds trusted to ship unreviewed (browsers, large -# vendors, Omacom's own projects). This workflow makes each new branch tip a commit pin -# in the recipe, which publish.yml then treats like any other version bump: -# the PR builds on the droplets, auto-merge lands it when `result` is green, -# and the merge publishes the artifacts. A tip that fails to build stays an -# unmerged red PR that the next tick supersedes. +# The hourly lane. Packages marked "auto_merge": true update here instead of +# in sync-upstream.yml's 6-hourly run: some follow a moving upstream branch +# (omarchy-dev and omarchy-settings-dev on quattro, omasnap-git on main) +# rather than tagged releases, so nothing in this repository changes when +# their source does; the rest are release feeds worth shipping within the +# hour (browsers, large vendors, Omacom's own projects). This workflow makes +# each new branch tip a commit pin in the recipe, which publish.yml then +# treats like any other version bump: the PR builds on the droplets, +# auto-merge lands it when `result` is green, and the merge publishes the +# artifacts. A tip that fails to build stays an unmerged red PR that the next +# tick supersedes. +# +# One PR per package, on auto/track-branches-, opened by +# package-prs.yml. Packages pinned from the same upstream branch +# (omarchy-dev and omarchy-settings-dev) share one, because sync-upstream +# moves them in lockstep. # # The PR and auto-merge use the PAT in PKGS_BOT_TOKEN so they trigger the # build and publish workflows. The built-in GITHUB_TOKEN cannot drive this @@ -18,8 +24,10 @@ name: Track upstream branches on: schedule: - # Every 2 hours, off the hour to dodge the scheduling backlog at :00 - - cron: '35 */2 * * *' + # Hourly, off the hour to dodge the scheduling backlog at :00. GitHub + # drops scheduled runs under load: the old every-2-hours entry fired + # every 6 to 9 hours in practice. + - cron: '35 * * * *' workflow_dispatch: inputs: packages: @@ -27,8 +35,8 @@ on: required: false default: '' -# One tracker at a time: two runs racing on auto/track-branches would each -# force-push their own pin over the other's. +# One tracker at a time: two runs racing on one auto/track-branches-* branch +# would each force-push their own pin over the other's. concurrency: group: track-branches cancel-in-progress: false @@ -38,6 +46,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + outputs: + groups: ${{ steps.groups.outputs.groups }} steps: - name: Require the tracking token @@ -54,7 +64,7 @@ jobs: with: persist-credentials: false - # Same container as the reviewed sync: vercmp decides whether a pin is + # Same container as sync-upstream.yml: vercmp decides whether a pin is # an upgrade with the comparator pacman uses on users' machines. - name: Pin tracked branches to their current tips id: sync @@ -88,71 +98,50 @@ jobs: PACKAGES: ${{ github.event.inputs.packages }} UPSTREAM_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check for changes + - name: Group the pins into PRs if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }} - id: changes + id: groups run: | - if [ -z "$(git status --porcelain)" ]; then - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - git status --porcelain - { - echo "### Pinned" - git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' | grep -E '^\+(pkgver|_commit)=' | sed 's/^+/- /' - } >> "$GITHUB_STEP_SUMMARY" - fi + groups=$(.github/scripts/package-pr-groups.sh auto/track-branches "$RUNNER_TEMP/pins.patch") + echo "groups=$groups" >> "$GITHUB_OUTPUT" - # The PR title names what moved, so the merged history reads like a - # changelog: "Track upstream branches: omarchy-dev 4.0.0.r6520.g1a2b3c4, ...". - - name: Describe the pins - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: describe - run: | - title=$(git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' \ - | awk '/^\+\+\+ b\/pkgbuilds\//{split($2,p,"/"); pkg=p[3]} /^\+pkgver=/{sub(/^\+pkgver=/,""); printf "%s %s, ", pkg, $0}' \ - | sed 's/, $//') - echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" - - - name: Open or update the tracking PR - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: pr - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + - name: Hand the pins to the PR jobs + if: ${{ !cancelled() && steps.groups.outputs.groups != '' && steps.groups.outputs.groups != '[]' }} + uses: actions/upload-artifact@v4 with: - token: ${{ secrets.PKGS_BOT_TOKEN }} - commit-message: ${{ steps.describe.outputs.title }} - title: ${{ steps.describe.outputs.title }} - body: | - Automated update of packages on the unattended lane - (`"auto_merge": true` in `.omarchy/package.json`). Branch-tracking - packages now pin their branch tip; release-feed packages move to - their newest upstream release. Fresh releases and tips wait until - they are at least `min_release_age` old. + name: package-pins + path: ${{ runner.temp }}/pins.patch + retention-days: 1 - This PR auto-merges once the build checks pass. A failing build - leaves it open; the next tracker run replaces it with the newer tip. - branch: auto/track-branches - delete-branch: true - labels: automated + prs: + needs: track + if: ${{ !cancelled() && needs.track.outputs.groups != '' }} + permissions: + contents: read + uses: ./.github/workflows/package-prs.yml + with: + groups: ${{ needs.track.outputs.groups }} + prefix: auto/track-branches + title: Track upstream branches + body: | + Automated pin of a package that follows a moving upstream branch or a + trusted release feed (`"auto_merge": true` in `.omarchy/package.json`). + Its recipe now points at the newest upstream release, or the newest + branch tip whose commit timestamp is at least `min_release_age` old. - # Auto-merge, not a direct merge: branch protection still has to see - # `result`, `self-tests` and `build-isolation` green, and this lane - # inherits every rule the reviewed lane has except the human. - - name: Enable auto-merge - if: ${{ !cancelled() && steps.pr.outputs.pull-request-number != '' }} - env: - GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} - PR: ${{ steps.pr.outputs.pull-request-number }} - run: | - # Idempotent across re-runs of an updated PR: enabling twice errors. - if [[ "$(gh pr view "$PR" -R "${{ github.repository }}" --json autoMergeRequest --jq '.autoMergeRequest != null')" == true ]]; then - echo "auto-merge already enabled on #$PR" - exit 0 - fi - gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" + This PR auto-merges once the build checks pass. A failing build leaves + it open without holding back any other package; the next tracker run + replaces it with the newer tip. + secrets: + PKGS_BOT_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + notify: + needs: [track, prs] + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: - name: Notify Basecamp on failure - if: failure() && env.BASECAMP_CHATBOT_URL != '' + if: ${{ env.BASECAMP_CHATBOT_URL != '' }} env: BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} run: | diff --git a/README.md b/README.md index 1f0bc5cf..1064544e 100644 --- a/README.md +++ b/README.md @@ -383,27 +383,28 @@ or an existing test needs a maintainer to merge it. ## Automatic updates -Three scheduled workflows open PRs. Each takes a `packages` input for a manual -run on named packages; for the two sync workflows that run opens its own PR, -on `auto/sync-upstream-` or `auto/sync-rebuilds-`. +Three scheduled workflows open PRs, one per package, through +`package-prs.yml`. Each PR auto-merges when its build is green, so the open +ones are the failures. Each workflow takes a `packages` input for a manual run +on named packages. -| Workflow | Runs | Opens | Merges | +| Workflow | Runs | Opens one PR per package on | For | |---|---|---|---| -| `sync-upstream.yml` | every 6 hours | one PR on `auto/sync-upstream` with every new upstream release | **a maintainer merges it** | -| `sync-rebuilds.yml` | every 6 hours | one PR on `auto/sync-rebuilds` bumping `pkgrel` where a `rebuild_on` dependency moved | itself, when green | -| `track-branches.yml` | every 2 hours | one PR on `auto/track-branches` updating `"auto_merge": true` packages to their newest upstream release or branch tip | itself, when green | +| `sync-upstream.yml` | every 6 hours | `auto/sync-upstream-` | a new upstream release | +| `sync-rebuilds.yml` | every 6 hours | `auto/sync-rebuilds-` | a `pkgrel` bump where a `rebuild_on` dependency moved | +| `track-branches.yml` | hourly | `auto/track-branches-` | `"auto_merge": true` packages: the newest upstream release or branch tip | -- **Each PR is a batch.** One package that fails to build keeps the whole PR - red and unmerged. Fix that package on `master`. For the two sync workflows, - a run with `packages` naming the healthy ones gives them their own PR. The - branch tracker has one PR only; its next run replaces it. +- **A red PR holds back only its own package.** Fix it on `master` or on the + PR. The next run updates the PR if upstream moves again, and closes it once + it conflicts with `master`. Packages pinned from the same upstream branch + (`omarchy-dev` and `omarchy-settings-dev`) share a PR, because they move in + lockstep. - **A package with no upstream declaration or hook gets no upstream updates.** That includes the kernels (`linux-omarchy*`, `linux-ptl`, `linux-aurora`): bump them by PR. - A failed run of any of the three posts to Basecamp when `BASECAMP_CHATBOT_URL` is set. -- Every self-merging path (`sync-rebuilds.yml`, `track-branches.yml` and - `auto-merge-pr.yml`) needs +- Every self-merging path (the three above and `auto-merge-pr.yml`) needs the `PKGS_BOT_TOKEN` secret: a personal access token with Contents and Pull requests write access to this repository, owned by an account trusted to build. A merge made with the built-in `GITHUB_TOKEN` does not start @@ -427,7 +428,7 @@ Details: [docs/upstream-sources.md](docs/upstream-sources.md), | `source` | Always `local`. | | `upstream` | Where releases come from. Mutually exclusive with an `.omarchy/upstream.sh` hook. | | `min_release_age` | Hold a new upstream release back this long (`"24h"`, `"2d"`). A release whose age cannot be proven fails the sync. | -| `auto_merge` | `true` moves the package's updates from the reviewed sync PR to `track-branches.yml`. For packages that follow a moving branch, and for trusted vendor and Omacom release feeds. Needs an upstream declaration. | +| `auto_merge` | `true` moves the package's updates from `sync-upstream.yml` (every 6 hours) to `track-branches.yml` (hourly). For packages that follow a moving branch, and for feeds worth shipping within the hour. Needs an upstream declaration. | | `release_ring` | `fast`: publish to `rc` and `stable` on merge, not only `edge`. Takes effect with the package's next version: bump `pkgrel` in the same PR. | | `channels` | The only channels the package may be published to. `omarchy-dev` is held to `["edge"]` this way. | | `pinned` | Version is set per release on the `rc` branch. Used by `omarchy` and `omarchy-settings`. | @@ -491,7 +492,7 @@ docs/ Reference | `approve-pr.yml` | PR events | Releases GitHub's hold on a `build-approved` PR's runs | | `sync-upstream.yml` | every 6 hours, dispatch | Upstream release PR | | `sync-rebuilds.yml` | every 6 hours, dispatch | Dependency rebuild PR | -| `track-branches.yml` | every 2 hours, dispatch | Branch tip PR | +| `track-branches.yml` | hourly, dispatch | Branch tip PRs | | `builder-images.yml` | daily, changes to the image's inputs (push and PR), dispatch | Builds and tests the builder image; publishes it from `master` | Secrets: the `publish` environment (usable from `master` only) holds the diff --git a/docs/rebuild-triggers.md b/docs/rebuild-triggers.md index a095c681..0954216f 100644 --- a/docs/rebuild-triggers.md +++ b/docs/rebuild-triggers.md @@ -37,7 +37,8 @@ A dependency this repository carries for an architecture (a recipe here that bui ## The workflow -`sync-rebuilds.yml` runs `bin/sync-rebuilds` every 6 hours, opens one PR on -`auto/sync-rebuilds` and enables auto-merge. The PR lands once `result`, -`self-tests` and `build-isolation` pass, and the merge publishes. A rebuild that -fails stays an open red PR for a maintainer. +`sync-rebuilds.yml` runs `bin/sync-rebuilds` every 6 hours and opens one PR per +package on `auto/sync-rebuilds-` with auto-merge enabled. Each PR lands +once `result`, `self-tests` and `build-isolation` pass, and the merge publishes. +A rebuild that fails stays an open red PR for a maintainer without holding back +the others. diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index b2474693..8dbac5ee 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -73,13 +73,18 @@ because its published history counted every commit and the number must never go down. `min_release_age` holds a branch tip until its commit timestamp is old enough. -A fresh tip leaves the existing pin alone; the watch never walks backward to -an older commit. This uses Git's committer date, not the time a commit was -pushed. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold. +A fresh tip falls back to the newest commit on the branch's first-parent +history that has cleared the window, so a busy branch still advances; it never +selects a commit from a merged side branch. This uses Git's committer date, not +the time a commit was pushed, measured from when the run cloned the branch so +packages pinned from it agree. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold. Packages marked `"auto_merge": true` ride the unattended lane -(`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened -and auto-merged as soon as the build checks pass. The lane works with every +(`track-branches.yml`), checked hourly instead of every 6 hours. Every lane +opens its PRs through `package-prs.yml` and auto-merges them as soon as the +build checks pass. Each package gets its own +PR on `auto/track-branches-`, and packages that pin the same branch +share one, so a package whose build fails holds back only itself. The lane works with every provider, not only branch watches, so it also carries release feeds trusted to ship without review: the browsers, large vendors' apps, Omacom's own projects, and vendor binaries that hold fresh releases with `min_release_age`. `bin/sync-upstream --lane @@ -224,22 +229,20 @@ These packages were already excluded from automatic AUR updates. The migration p - Spotify uses HTTPS and retains its signed Release/Packages verification. - Source and build compatibility still need review when upstream code changes. Direct watches remove AUR recipe churn, not the need to maintain packaging. -## How the sync PR builds +## How the sync PRs build -`sync-upstream.yml` pushes its PR with the built-in `GITHUB_TOKEN`, so GitHub -holds the PR's build and test runs for approval. The workflow labels its own -PR `build-approved`, and its `approve` job releases the held runs for each -commit it pushes. A push to an `auto/sync-*` branch, or to the tracker's -`auto/track-branches`, does not cancel the PR's build in flight: the new +`sync-upstream.yml`, `sync-rebuilds.yml` and `track-branches.yml` each run their +sync once, split the changed recipes into one PR per package with +`.github/scripts/package-pr-groups.sh`, and hand them to `package-prs.yml`. That +pushes each PR with the PAT in `PKGS_BOT_TOKEN`, so its builds start without an +approval hold, and arms auto-merge. A push to an `auto/sync-*` or +`auto/track-branches-*` branch does not cancel the PR's build in flight: the new build waits, then reuses every artifact the finished one uploaded. -Because of that push, GitHub starts no `pull_request_target` workflow for the -PR, so `auto-merge-pr.yml` never arms it: a maintainer merges it. - -A run started by hand with `packages` opens its own PR on -`auto/sync-upstream-`. The next scheduled run still picks the same -update up in the shared PR if it has not merged by then; identical package -trees reuse the same build artifacts. +A red PR stays open for a maintainer and holds back only its own package. A +later run updates it if upstream moves again, and closes it once it conflicts +with `master`. A run started by hand with `packages` updates the same +per-package PRs a scheduled run would. ## Declarative providers and hooks diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index 7ab28a09..992042ca 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -186,9 +186,9 @@ def matches(watch, text, extra=None, full=False): yield candidate(watch, {**(extra or {}), **match.groupdict()}) -def git_branch_tip(url, branch, tag_pattern, cache): - """Describe the current tip of an upstream branch: - commit, total count, date, and with a tag_pattern +def git_branch_tip(url, branch, tag_pattern, cache, min_age=0): + """Describe the newest commit on an upstream branch that has cleared + min_age: commit, total count, date, and with a tag_pattern the newest release tag reachable from it plus the distance from that tag, so a branch build can be versioned .r.g, above the release it follows and below the next one, the way a pkgver() function would. @@ -196,20 +196,33 @@ def git_branch_tip(url, branch, tag_pattern, cache): One blobless single-branch clone per (url, branch) per run, shared by every package that tracks it, so two recipes pinned from one clone always see the same commit. The clone is read with git only; nothing in it runs. - select_release applies the age hold to this tip, without walking back - into history (which could select a commit from a merged side branch). + A tip younger than min_age falls back along first-parent history, the + commits that were themselves the branch tip, never into a merged side + branch. Holding the whole branch instead stalled a busy branch for as + long as it kept getting commits. Age is measured from when the clone was + made, so siblings synced seconds apart cannot straddle the window. """ https(url) key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest() work = Path(cache) / f"{key}.branch.git" + cloned_at = work.with_name(f"{key}.branch.time") if not work.exists(): scratch = work.with_name(f"{work.name}.{os.getpid()}.tmp") + started = dt.datetime.now(dt.timezone.utc).isoformat() subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True) + cloned_at.write_text(started) scratch.replace(work) git = ["git", "-C", str(work)] commit = run([*git, "rev-parse", "HEAD"], text=True).strip() if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("branch tip is not a commit") + if min_age: + now = dt.datetime.fromisoformat(cloned_at.read_text().strip()) + for line in run([*git, "log", "--first-parent", "--format=%H %cI", commit], text=True).splitlines(): + sha, committed = line.split() + if (now - dt.datetime.fromisoformat(committed)).total_seconds() >= min_age: + commit = sha + break count = run([*git, "rev-list", "--count", commit], text=True).strip() date = run([*git, "show", "-s", "--format=%cs", commit], text=True).strip().replace("-", "") timestamp = run([*git, "show", "-s", "--format=%cI", commit], text=True).strip() @@ -233,7 +246,7 @@ def git_branch_tip(url, branch, tag_pattern, cache): return values -def discover(watch, fetch): +def discover(watch, fetch, min_age=0): provider = validate(watch) feed = watch[provider] results = [] @@ -260,7 +273,7 @@ def discover(watch, fetch): for tag, commit in tags.items(): results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True)) elif provider == "git_branch": - tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache) + tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age) results.append(candidate(watch, tip)) elif provider == "npm": data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe="")) @@ -544,7 +557,7 @@ def sync(package, fetch, min_age=0, check=False): original = path.read_text() before = read_recipe(path) bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1" - release = select_release(discover(watch, fetch), min_age, bypass=bypass) + release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass) if release is None: return {"status": "skipped", "reason": "minimum release age"} current = scalar(before, "pkgver") diff --git a/tests/pr-workflow-approval.cjs b/tests/pr-workflow-approval.cjs index d72c16b1..10fcb615 100644 --- a/tests/pr-workflow-approval.cjs +++ b/tests/pr-workflow-approval.cjs @@ -1,5 +1,5 @@ const assert = require('node:assert/strict'); -const { readFileSync, mkdtempSync, rmSync } = require('node:fs'); +const { readFileSync, writeFileSync, mkdirSync, mkdtempSync, rmSync } = require('node:fs'); const { join } = require('node:path'); const { tmpdir } = require('node:os'); const { test } = require('node:test'); @@ -333,82 +333,12 @@ test('only same-repository sync branches queue behind an in-flight build', () => assert.equal(cancels('omacom/omarchy-pkgs', 'auto/sync-upstream'), false); assert.equal(cancels('omacom/omarchy-pkgs', 'auto/sync-upstream-ttfx'), false); assert.equal(cancels('omacom/omarchy-pkgs', 'auto/sync-rebuilds'), false); + assert.equal(cancels('omacom/omarchy-pkgs', 'auto/track-branches-omarchy-dev-omarchy-settings-dev'), false); assert.equal(cancels('omacom/omarchy-pkgs', 'ttfx/fix'), true); assert.equal(cancels('someone/omarchy-pkgs', 'auto/sync-upstream'), true); assert.equal(cancels(undefined, ''), true); // workflow_dispatch }); -// The sync workflows release their own GITHUB_TOKEN pushes: GitHub creates -// no pull_request_target run for those, so approve-pr.yml never runs. -const approveSyncPush = require('../.github/scripts/approve-sync-push.cjs'); -function syncFixture(options = {}) { - const f = fixture([run(1, BUILD, { head_branch: 'auto/sync-upstream' }), - run(2, TESTS, { head_branch: 'auto/sync-upstream' })], options); - Object.assign(f.state.pr, { - user: { login: 'github-actions[bot]' }, - head: { ...f.state.pr.head, ref: 'auto/sync-upstream', repo: { id: 42, full_name: 'omacom/omarchy-pkgs' } }, - base: { repo: { full_name: 'omacom/omarchy-pkgs' } }, - }); - const push = overrides => approveSyncPush({ - github: f.github, context: { repo: { owner: 'omacom', repo: 'omarchy-pkgs' }, payload: {} }, - core: { info() {} }, number: 390, branch: 'auto/sync-upstream', headSha: pr.head.sha, - since: earlier, attempts: 6, sleep: async () => {}, ...overrides, - }); - return { ...f, push }; -} - -test('a labelled sync PR has its bot push released', async () => { - const { state, push } = syncFixture(); - await push(); - assert.deepEqual(state.approved, [1, 2]); -}); - -test('a sync PR whose label a maintainer removed stays held', async () => { - const { state, push } = syncFixture(); - state.pr.labels = []; - await push(); - assert.deepEqual(state.approved, []); -}); - -test('runs older than the push are not taken for this push', async () => { - const { state, push } = syncFixture(); - await assert.rejects(push({ since: '2026-09-19T03:00:00Z' }), /Timed out/); - assert.deepEqual(state.approved, []); -}); - -for (const [name, change] of [ - ['a contributor PR', current => { current.user.login = 'someone'; }], - ['a fork PR', current => { current.head.repo.full_name = 'someone/omarchy-pkgs'; }], - ['another branch', current => { current.head.ref = 'auto/sync-rebuilds'; }], -]) { - test(`the sync approver refuses ${name}, even when labelled`, async () => { - const { state, push } = syncFixture(); - change(state.pr); - await assert.rejects(push(), /refusing to approve/); - assert.deepEqual(state.approved, []); - }); -} - -for (const [name, change] of [ - ['closed', current => { current.state = 'closed'; }], - ['moved on', current => { current.head.sha = 'newer-sha'; }], -]) { - test(`a sync PR that has ${name} is left alone`, async () => { - const { state, push } = syncFixture(); - change(state.pr); - await push(); - assert.deepEqual(state.approved, []); - }); -} - -test('the sync approver needs the push it is approving for', async () => { - const { state, push } = syncFixture(); - for (const missing of [{ number: NaN }, { headSha: '' }, { since: '' }, { branch: '' }]) { - await assert.rejects(push(missing), /Missing sync PR/); - } - assert.deepEqual(state.approved, []); -}); - // A scoped dispatch must not push to the shared branch: it would replace the // other pending updates in the open sync PR with just the named packages. const branchScript = join(__dirname, '../.github/scripts/sync-pr-branch.sh'); @@ -434,35 +364,66 @@ test('scoped branch names reject anything that is not a package name', () => { } }); -test('sync workflows push scoped runs aside and keep actions: write out of the sync job', () => { - for (const file of ['sync-upstream.yml']) { +// Every bot lane opens one PR per package through package-prs.yml, with the +// PAT: a GITHUB_TOKEN push would wait for build approval, and a GITHUB_TOKEN +// merge would not start publish.yml. +test('every bot lane opens its PRs per package through package-prs.yml', () => { + for (const [file, prefix] of [['sync-upstream.yml', 'auto/sync-upstream'], + ['sync-rebuilds.yml', 'auto/sync-rebuilds'], ['track-branches.yml', 'auto/track-branches']]) { const text = readFileSync(join(__dirname, '../.github/workflows', file), 'utf8'); - const sync = text.slice(text.indexOf('\n sync:\n'), text.indexOf('\n approve:\n')); - const approveJob = text.slice(text.indexOf('\n approve:\n')); - assert.match(sync, /sync-pr-branch\.sh auto\/sync-[\w-]+ "\$\{package_args\[@\]\}"/, file); - assert.match(sync, /branch: \$\{\{ steps\.branch\.outputs\.branch \}\}/, file); - assert.doesNotMatch(sync, /^ +actions: write$/m, file); - assert.match(approveJob, /^ actions: write$/m, file); - // The sync PR is bot-authored and trusted: it labels itself so its own - // pushes build, including the push that opens the PR. - assert.match(sync, /labels: \|\n +automated\n +build-approved\n/, file); - assert.match(approveJob, /needs\.sync\.outputs\.operation == 'created'/, file); - assert.match(approveJob, /needs\.sync\.outputs\.operation == 'updated'/, file); + assert.match(text, new RegExp(`package-pr-groups\\.sh ${prefix.replace('/', '\\/')} "\\$RUNNER_TEMP\/pins\\.patch"`), file); + assert.match(text, /uses: \.\/\.github\/workflows\/package-prs\.yml/, file); + assert.match(text, new RegExp(`prefix: ${prefix.replace('/', '\\/')}\n`), file); + assert.match(text, /PKGS_BOT_TOKEN: \$\{\{ secrets\.PKGS_BOT_TOKEN \}\}/, file); + assert.doesNotMatch(text, /create-pull-request|build-approved|\n approve:\n/, file); + assert.doesNotMatch(text, /^ +(contents|pull-requests|actions): write$/m, file); } + const prs = readFileSync(join(__dirname, '../.github/workflows/package-prs.yml'), 'utf8'); + assert.match(prs, /token: \$\{\{ secrets\.PKGS_BOT_TOKEN \}\}/); + assert.doesNotMatch(prs, /secrets\.GITHUB_TOKEN/); + assert.match(prs, /gh pr merge --auto --merge "\$PR"/); + assert.match(prs, /fail-fast: false/); + assert.doesNotMatch(prs, /^ +(contents|pull-requests|actions): write$/m); }); -test('rebuild sync opens its PR with the bot token and auto-merges it', () => { - const file = 'sync-rebuilds.yml'; - const text = readFileSync(join(__dirname, '../.github/workflows', file), 'utf8'); - assert.match(text, /sync-pr-branch\.sh auto\/sync-rebuilds "\$\{package_args\[@\]\}"/, file); - assert.match(text, /branch: \$\{\{ steps\.branch\.outputs\.branch \}\}/, file); - // A GITHUB_TOKEN merge would not start publish.yml, so the PR and the - // merge both go through the PAT. - assert.match(text, /token: \$\{\{ secrets\.PKGS_BOT_TOKEN \}\}/, file); - assert.doesNotMatch(text, /secrets\.GITHUB_TOKEN/, file); - assert.match(text, /gh pr merge --auto --merge "\$PR"/, file); - assert.doesNotMatch(text, /^ +actions: write$/m, file); - assert.doesNotMatch(text, /\n approve:\n/, file); +// One PR per package, except packages pinned from the same upstream branch, +// which move in lockstep and so share one. +const groupsScript = join(__dirname, '../.github/scripts/package-pr-groups.sh'); +test('package PRs split per package and keep branch siblings together', () => { + const dir = mkdtempSync(join(tmpdir(), 'package-pr-groups-')); + try { + const git = (...args) => execFileSync('git', ['-C', dir, '-c', 'user.name=T', '-c', 'user.email=t@t', ...args]); + git('init', '-q'); + const write = (name, watch) => { + const pkg = join(dir, 'pkgbuilds', name); + mkdirSync(join(pkg, '.omarchy'), { recursive: true }); + writeFileSync(join(pkg, 'PKGBUILD'), `pkgname=${name}\npkgver=1\npkgrel=1\n`); + writeFileSync(join(pkg, '.omarchy/package.json'), JSON.stringify({ upstream: { watch } })); + }; + const branch = { git_branch: 'https://github.com/basecamp/omarchy.git', branch: 'quattro' }; + write('omarchy-dev', branch); + write('omarchy-settings-dev', branch); + write('omasnap-git', { git_branch: 'https://github.com/omacom/omasnap.git', branch: 'main' }); + write('zed', { github: 'zed-industries/zed' }); + write('untouched', { github: 'x/y' }); + git('add', '.'); + git('commit', '-qm', 'base'); + for (const name of ['omarchy-dev', 'omarchy-settings-dev', 'omasnap-git', 'zed']) { + writeFileSync(join(dir, 'pkgbuilds', name, 'PKGBUILD'), `pkgname=${name}\npkgver=2\npkgrel=1\n`); + } + const patch = join(dir, 'pins.patch'); + const groups = JSON.parse(execFileSync(groupsScript, ['auto/track-branches', patch], { cwd: dir, encoding: 'utf8' })); + assert.deepEqual(groups, [ + { branch: 'auto/track-branches-omarchy-dev-omarchy-settings-dev', packages: 'omarchy-dev omarchy-settings-dev' }, + { branch: 'auto/track-branches-omasnap-git', packages: 'omasnap-git' }, + { branch: 'auto/track-branches-zed', packages: 'zed' }, + ]); + assert.match(readFileSync(patch, 'utf8'), /pkgbuilds\/zed\/PKGBUILD/); + git('checkout', '-q', '--', 'pkgbuilds'); + assert.deepEqual(JSON.parse(execFileSync(groupsScript, ['auto/sync-upstream', patch], { cwd: dir, encoding: 'utf8' })), []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } }); const { decide } = require('../.github/scripts/auto-merge-pr.cjs'); @@ -512,17 +473,14 @@ test('auto-merge accepts the test a package PR brings, and nothing else outside assert.equal(check([...packageFiles, wiring(undefined)]), false); }); -test('auto-merge skips drafts, closed PRs and the reviewed upstream sync', () => { +test('auto-merge skips drafts and closed PRs, and arms every bot lane', () => { const check = pr => decide({ pr, files: packageFiles, vouchStatus: 'bot', repository }).enable; assert.equal(check(mergeable({ draft: true })), false); assert.equal(check(mergeable({ state: 'closed' })), false); - for (const ref of ['auto/sync-upstream', 'auto/sync-upstream/walker']) { - assert.equal(check(mergeable({ head: { ref, repo: { full_name: repository } } })), false, ref); + // The bot lanes already enable their own auto-merge; agreeing is harmless. + for (const ref of ['auto/sync-upstream-zed', 'auto/sync-rebuilds-walker', 'auto/track-branches-omarchy-dev-omarchy-settings-dev']) { + assert.equal(check(mergeable({ head: { ref, repo: { full_name: repository } } })), true, ref); } - // The unattended lanes already enable their own auto-merge; agreeing is harmless. - assert.equal(check(mergeable({ head: { ref: 'auto/sync-rebuilds', repo: { full_name: repository } } })), true); - // A fork's branch named like ours is just a contributor branch. - assert.equal(check(mergeable({ head: { ref: 'auto/sync-upstream', repo: { full_name: 'someone/omarchy-pkgs' } } })), true); }); test('auto-merge workflow enables with the bot token and never checks out the PR', () => { diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index 65dfd9a4..b40616af 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -250,14 +250,41 @@ b2sums=('old' 'local-b2') with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'): w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P[0-9.]+)', self.root / 'other-cache') - def test_git_branch_min_age_holds_the_tip_instead_of_selecting_history(self): + def test_git_branch_min_age_falls_back_to_the_newest_old_enough_tip(self): repo, shas = self.branch_fixture(fresh_tip=True) watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} with self.redirect_clone(repo): releases = w.discover(watch, self.fetch) + held = w.discover(watch, self.fetch, min_age=3600) self.assertEqual(w.select_release(releases)['values']['commit'], shas[-1]) + # The backstop still refuses the fresh tip itself... self.assertIsNone(w.select_release(releases, min_age=3600)) self.assertEqual(w.select_release(releases, min_age=3600, bypass=True)['values']['commit'], shas[-1]) + # ...and the watch offers the commit before it instead of nothing. + self.assertEqual(w.select_release(held, min_age=3600)['values']['commit'], shas[-2]) + self.assertEqual(held[0]['values']['count'], str(len(shas) - 1)) + + def test_git_branch_min_age_never_falls_back_into_a_merged_side_branch(self): + repo, shas = self.branch_fixture() + git = ['git', '-C', str(repo), '-c', 'user.name=Test', '-c', 'user.email=test@example.test'] + old = {**os.environ, 'GIT_COMMITTER_DATE': '2020-01-02T00:00:00+00:00', 'GIT_AUTHOR_DATE': '2020-01-02T00:00:00+00:00'} + subprocess.run([*git, 'checkout', '-q', '-b', 'side'], check=True) + (repo / 'side').write_text('x') + subprocess.run([*git, 'add', '.'], check=True) + subprocess.run([*git, 'commit', '-qm', 'side'], env=old, check=True) + subprocess.run([*git, 'checkout', '-q', 'main'], check=True) + subprocess.run([*git, 'merge', '-q', '--no-ff', '-m', 'merge side', 'side'], check=True) + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + with self.redirect_clone(repo): + held = w.discover(watch, self.fetch, min_age=3600) + self.assertEqual(w.select_release(held, min_age=3600)['values']['commit'], shas[-1]) + + def test_git_branch_min_age_holds_when_nothing_is_old_enough(self): + repo, shas = self.branch_fixture(fresh_tip=True) + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + with self.redirect_clone(repo): + held = w.discover(watch, self.fetch, min_age=10 ** 10) + self.assertIsNone(w.select_release(held, min_age=10 ** 10)) def branch_sync_fixture(self): repo, shas = self.branch_fixture()