diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 611d373d..ed0a70b1 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -21,7 +21,7 @@ on: required: true # A new push normally cancels the PR's in-flight build. The bots' branches -# (auto/sync-* and auto/track-branches) are the exception: they are +# (auto/sync-* and auto/track-branches-*) are the exception: they are # force-pushed with fresh upstream releases several times a day, which kept # cancelling multi-hour aarch64 builds before they could finish. There the # newest run waits instead (GitHub keeps at most one pending run per group, @@ -31,7 +31,7 @@ on: # runs at once; cancelling the first left a red `result` on the PR (#901). concurrency: group: build-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ !(github.event.pull_request.head.repo.full_name == github.repository && (startsWith(github.head_ref, 'auto/sync-') || github.head_ref == 'auto/track-branches')) }} + cancel-in-progress: ${{ !(github.event.pull_request.head.repo.full_name == github.repository && (startsWith(github.head_ref, 'auto/sync-') || startsWith(github.head_ref, 'auto/track-branches'))) }} jobs: # Builds cost real machines, so they run only for trusted authors: diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml index 2894d36e..c1f88e04 100644 --- a/.github/workflows/track-branches.yml +++ b/.github/workflows/track-branches.yml @@ -11,6 +11,12 @@ name: Track upstream branches # and the merge publishes the artifacts. A tip that fails to build stays an # unmerged red PR that the next tick supersedes. # +# One PR per package, on auto/track-branches-. Packages pinned from +# the same upstream branch (omarchy-dev and omarchy-settings-dev) share one, +# because sync-upstream moves them in lockstep. A single batch PR let one +# package with a broken build hold every other package back: omasnap-git's +# smoke suite kept the quattro pins unmerged for four days (#788). +# # The PR and auto-merge use the PAT in PKGS_BOT_TOKEN so they trigger the # build and publish workflows. The built-in GITHUB_TOKEN cannot drive this # unattended chain. The PAT needs Contents: write and Pull requests: write @@ -18,8 +24,10 @@ name: Track upstream branches on: schedule: - # Every 2 hours, off the hour to dodge the scheduling backlog at :00 - - cron: '35 */2 * * *' + # Hourly, off the hour to dodge the scheduling backlog at :00. GitHub + # drops scheduled runs under load: the old every-2-hours entry fired + # every 6 to 9 hours in practice. + - cron: '35 * * * *' workflow_dispatch: inputs: packages: @@ -27,8 +35,8 @@ on: required: false default: '' -# One tracker at a time: two runs racing on auto/track-branches would each -# force-push their own pin over the other's. +# One tracker at a time: two runs racing on one auto/track-branches-* branch +# would each force-push their own pin over the other's. concurrency: group: track-branches cancel-in-progress: false @@ -38,6 +46,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + outputs: + groups: ${{ steps.groups.outputs.groups }} steps: - name: Require the tracking token @@ -88,25 +98,101 @@ jobs: PACKAGES: ${{ github.event.inputs.packages }} UPSTREAM_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Check for changes + # The PR a pin rides: one per package, or one per upstream branch for + # packages that must move together. The pins travel to the PR jobs as + # a patch, so the upstream feeds are read once and every PR is cut + # from the same answers. + - name: Group the pins into PRs if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }} - id: changes + id: groups run: | - if [ -z "$(git status --porcelain)" ]; then - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "has_changes=true" >> "$GITHUB_OUTPUT" - git status --porcelain - { - echo "### Pinned" - git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' | grep -E '^\+(pkgver|_commit)=' | sed 's/^+/- /' - } >> "$GITHUB_STEP_SUMMARY" - fi + git diff --binary -- pkgbuilds > "$RUNNER_TEMP/pins.patch" + declare -A members=() + while IFS= read -r package; do + key=$(jq -r '(.upstream.watch? | objects | select(has("git_branch")) | "\(.git_branch)#\(.branch)") // empty' \ + "pkgbuilds/$package/.omarchy/package.json" 2>/dev/null || true) + members["${key:-$package}"]+="$package " + done < <(git diff --name-only -- pkgbuilds | cut -d/ -f2 | sort -u) + groups='[]' + for key in "${!members[@]}"; do + read -r -a packages <<<"${members[$key]}" + branch=$(.github/scripts/sync-pr-branch.sh auto/track-branches "${packages[@]}" | sed -n 's/^branch=//p') + groups=$(jq -c --arg branch "$branch" --arg packages "${packages[*]}" \ + '. + [{branch: $branch, packages: $packages}] | sort_by(.branch)' <<<"$groups") + done + echo "groups=$groups" >> "$GITHUB_OUTPUT" + jq -r '.[] | "- `\(.branch)`: \(.packages)"' <<<"$groups" | sed '1i ### Tracker PRs' >> "$GITHUB_STEP_SUMMARY" + + - name: Hand the pins to the PR jobs + if: ${{ !cancelled() && steps.groups.outputs.groups != '' && steps.groups.outputs.groups != '[]' }} + uses: actions/upload-artifact@v4 + with: + name: pins + path: ${{ runner.temp }}/pins.patch + retention-days: 1 + + # A tracker PR that this run did not refresh and that conflicts with + # master is stale: master moved its packages on without it. Close it + # rather than leave a red PR behind. Whether a PR was refreshed is not + # enough on its own, since a feed that failed this run (one of fifty + # timing out is routine) leaves a still-valid PR unrefreshed too. The + # old single auto/track-branches PR is retired outright: the per-package + # PRs replace it. + - name: Close superseded tracker PRs + if: ${{ !cancelled() && steps.groups.outcome == 'success' }} + env: + GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} + GROUPS_JSON: ${{ steps.groups.outputs.groups }} + run: | + gh pr list -R "${{ github.repository }}" --state open --limit 200 --json number,headRefName,mergeable \ + --jq '.[] | select(.headRefName | startswith("auto/track-branches")) | "\(.number) \(.headRefName) \(.mergeable)"' | + while read -r number branch mergeable; do + jq -e --arg branch "$branch" 'any(.[]; .branch == $branch)' <<<"$GROUPS_JSON" >/dev/null && continue + [[ $branch == auto/track-branches || $mergeable == CONFLICTING ]] || continue + echo "closing #$number ($branch): superseded" + gh pr close "$number" -R "${{ github.repository }}" --delete-branch \ + --comment "Superseded: master has moved these packages on, or a per-package tracker PR carries them now." + done + + pr: + needs: track + if: ${{ !cancelled() && needs.track.outputs.groups != '' && needs.track.outputs.groups != '[]' }} + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + max-parallel: 4 + matrix: + group: ${{ fromJSON(needs.track.outputs.groups) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Fetch the pins + uses: actions/download-artifact@v4 + with: + name: pins + path: ${{ runner.temp }} + + - name: Apply this PR's pins + env: + PACKAGES: ${{ matrix.group.packages }} + run: | + read -r -a packages <<<"$PACKAGES" + git apply $(printf -- '--include=pkgbuilds/%s/* ' "${packages[@]}") "$RUNNER_TEMP/pins.patch" + git status --porcelain + { + echo "### Pinned" + git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' | grep -E '^\+(pkgver|_commit)=' | sed 's/^+/- /' + } >> "$GITHUB_STEP_SUMMARY" # The PR title names what moved, so the merged history reads like a # changelog: "Track upstream branches: omarchy-dev 4.0.0.r6520.g1a2b3c4, ...". - name: Describe the pins - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} id: describe run: | title=$(git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' \ @@ -115,7 +201,6 @@ jobs: echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" - name: Open or update the tracking PR - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} id: pr uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: @@ -123,15 +208,16 @@ jobs: commit-message: ${{ steps.describe.outputs.title }} title: ${{ steps.describe.outputs.title }} body: | - Automated update of packages on the unattended lane - (`"auto_merge": true` in `.omarchy/package.json`). Branch-tracking - packages now pin their branch tip; release-feed packages move to - their newest upstream release. Fresh releases and tips wait until - they are at least `min_release_age` old. + Automated pin of packages that follow a moving upstream branch + or a trusted release feed (`"auto_merge": true` in + `.omarchy/package.json`). Each package's recipe now points at the + newest upstream release, or the newest branch tip whose commit + timestamp is at least `min_release_age` old. This PR auto-merges once the build checks pass. A failing build - leaves it open; the next tracker run replaces it with the newer tip. - branch: auto/track-branches + leaves it open without holding back any other package; the next + tracker run replaces it with the newer tip. + branch: ${{ matrix.group.branch }} delete-branch: true labels: automated @@ -139,7 +225,7 @@ jobs: # `result`, `self-tests` and `build-isolation` green, and this lane # inherits every rule the reviewed lane has except the human. - name: Enable auto-merge - if: ${{ !cancelled() && steps.pr.outputs.pull-request-number != '' }} + if: ${{ steps.pr.outputs.pull-request-number != '' }} env: GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} PR: ${{ steps.pr.outputs.pull-request-number }} @@ -151,8 +237,13 @@ jobs: fi gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" + notify: + needs: [track, pr] + if: ${{ failure() }} + runs-on: ubuntu-latest + steps: - name: Notify Basecamp on failure - if: failure() && env.BASECAMP_CHATBOT_URL != '' + if: ${{ env.BASECAMP_CHATBOT_URL != '' }} env: BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} run: | diff --git a/README.md b/README.md index 1f0bc5cf..d6da7ea7 100644 --- a/README.md +++ b/README.md @@ -391,12 +391,13 @@ on `auto/sync-upstream-` or `auto/sync-rebuilds-`. |---|---|---|---| | `sync-upstream.yml` | every 6 hours | one PR on `auto/sync-upstream` with every new upstream release | **a maintainer merges it** | | `sync-rebuilds.yml` | every 6 hours | one PR on `auto/sync-rebuilds` bumping `pkgrel` where a `rebuild_on` dependency moved | itself, when green | -| `track-branches.yml` | every 2 hours | one PR on `auto/track-branches` updating `"auto_merge": true` packages to their newest upstream release or branch tip | itself, when green | +| `track-branches.yml` | hourly | one PR per package on `auto/track-branches-` updating `"auto_merge": true` packages to their newest upstream release or branch tip; packages pinned from the same branch share one | itself, when green | -- **Each PR is a batch.** One package that fails to build keeps the whole PR - red and unmerged. Fix that package on `master`. For the two sync workflows, - a run with `packages` naming the healthy ones gives them their own PR. The - branch tracker has one PR only; its next run replaces it. +- **Each sync PR is a batch.** One package that fails to build keeps the + whole PR red and unmerged. Fix that package on `master`. A run with + `packages` naming the healthy ones gives them their own PR. The branch + tracker opens a PR per package, so a red one holds back only itself; its + next run replaces it, and closes it once master holds a newer pin. - **A package with no upstream declaration or hook gets no upstream updates.** That includes the kernels (`linux-omarchy*`, `linux-ptl`, `linux-aurora`): bump them by PR. @@ -491,7 +492,7 @@ docs/ Reference | `approve-pr.yml` | PR events | Releases GitHub's hold on a `build-approved` PR's runs | | `sync-upstream.yml` | every 6 hours, dispatch | Upstream release PR | | `sync-rebuilds.yml` | every 6 hours, dispatch | Dependency rebuild PR | -| `track-branches.yml` | every 2 hours, dispatch | Branch tip PR | +| `track-branches.yml` | hourly, dispatch | Branch tip PRs | | `builder-images.yml` | daily, changes to the image's inputs (push and PR), dispatch | Builds and tests the builder image; publishes it from `master` | Secrets: the `publish` environment (usable from `master` only) holds the diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index b2474693..ad0ff24b 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -73,13 +73,17 @@ because its published history counted every commit and the number must never go down. `min_release_age` holds a branch tip until its commit timestamp is old enough. -A fresh tip leaves the existing pin alone; the watch never walks backward to -an older commit. This uses Git's committer date, not the time a commit was -pushed. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold. +A fresh tip falls back to the newest commit on the branch's first-parent +history that has cleared the window, so a busy branch still advances; it never +selects a commit from a merged side branch. This uses Git's committer date, not +the time a commit was pushed, measured from when the run cloned the branch so +packages pinned from it agree. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold. Packages marked `"auto_merge": true` ride the unattended lane (`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened -and auto-merged as soon as the build checks pass. The lane works with every +and auto-merged as soon as the build checks pass. Each package gets its own +PR on `auto/track-branches-`, and packages that pin the same branch +share one, so a package whose build fails holds back only itself. The lane works with every provider, not only branch watches, so it also carries release feeds trusted to ship without review: the browsers, large vendors' apps, Omacom's own projects, and vendor binaries that hold fresh releases with `min_release_age`. `bin/sync-upstream --lane @@ -229,8 +233,8 @@ These packages were already excluded from automatic AUR updates. The migration p `sync-upstream.yml` pushes its PR with the built-in `GITHUB_TOKEN`, so GitHub holds the PR's build and test runs for approval. The workflow labels its own PR `build-approved`, and its `approve` job releases the held runs for each -commit it pushes. A push to an `auto/sync-*` branch, or to the tracker's -`auto/track-branches`, does not cancel the PR's build in flight: the new +commit it pushes. A push to an `auto/sync-*` branch, or to one of the +tracker's `auto/track-branches-*` branches, does not cancel the PR's build in flight: the new build waits, then reuses every artifact the finished one uploaded. Because of that push, GitHub starts no `pull_request_target` workflow for the diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index 7ab28a09..992042ca 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -186,9 +186,9 @@ def matches(watch, text, extra=None, full=False): yield candidate(watch, {**(extra or {}), **match.groupdict()}) -def git_branch_tip(url, branch, tag_pattern, cache): - """Describe the current tip of an upstream branch: - commit, total count, date, and with a tag_pattern +def git_branch_tip(url, branch, tag_pattern, cache, min_age=0): + """Describe the newest commit on an upstream branch that has cleared + min_age: commit, total count, date, and with a tag_pattern the newest release tag reachable from it plus the distance from that tag, so a branch build can be versioned .r.g, above the release it follows and below the next one, the way a pkgver() function would. @@ -196,20 +196,33 @@ def git_branch_tip(url, branch, tag_pattern, cache): One blobless single-branch clone per (url, branch) per run, shared by every package that tracks it, so two recipes pinned from one clone always see the same commit. The clone is read with git only; nothing in it runs. - select_release applies the age hold to this tip, without walking back - into history (which could select a commit from a merged side branch). + A tip younger than min_age falls back along first-parent history, the + commits that were themselves the branch tip, never into a merged side + branch. Holding the whole branch instead stalled a busy branch for as + long as it kept getting commits. Age is measured from when the clone was + made, so siblings synced seconds apart cannot straddle the window. """ https(url) key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest() work = Path(cache) / f"{key}.branch.git" + cloned_at = work.with_name(f"{key}.branch.time") if not work.exists(): scratch = work.with_name(f"{work.name}.{os.getpid()}.tmp") + started = dt.datetime.now(dt.timezone.utc).isoformat() subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True) + cloned_at.write_text(started) scratch.replace(work) git = ["git", "-C", str(work)] commit = run([*git, "rev-parse", "HEAD"], text=True).strip() if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("branch tip is not a commit") + if min_age: + now = dt.datetime.fromisoformat(cloned_at.read_text().strip()) + for line in run([*git, "log", "--first-parent", "--format=%H %cI", commit], text=True).splitlines(): + sha, committed = line.split() + if (now - dt.datetime.fromisoformat(committed)).total_seconds() >= min_age: + commit = sha + break count = run([*git, "rev-list", "--count", commit], text=True).strip() date = run([*git, "show", "-s", "--format=%cs", commit], text=True).strip().replace("-", "") timestamp = run([*git, "show", "-s", "--format=%cI", commit], text=True).strip() @@ -233,7 +246,7 @@ def git_branch_tip(url, branch, tag_pattern, cache): return values -def discover(watch, fetch): +def discover(watch, fetch, min_age=0): provider = validate(watch) feed = watch[provider] results = [] @@ -260,7 +273,7 @@ def discover(watch, fetch): for tag, commit in tags.items(): results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True)) elif provider == "git_branch": - tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache) + tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age) results.append(candidate(watch, tip)) elif provider == "npm": data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe="")) @@ -544,7 +557,7 @@ def sync(package, fetch, min_age=0, check=False): original = path.read_text() before = read_recipe(path) bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1" - release = select_release(discover(watch, fetch), min_age, bypass=bypass) + release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass) if release is None: return {"status": "skipped", "reason": "minimum release age"} current = scalar(before, "pkgver") diff --git a/tests/pr-workflow-approval.cjs b/tests/pr-workflow-approval.cjs index d72c16b1..d4b90515 100644 --- a/tests/pr-workflow-approval.cjs +++ b/tests/pr-workflow-approval.cjs @@ -333,6 +333,7 @@ test('only same-repository sync branches queue behind an in-flight build', () => assert.equal(cancels('omacom/omarchy-pkgs', 'auto/sync-upstream'), false); assert.equal(cancels('omacom/omarchy-pkgs', 'auto/sync-upstream-ttfx'), false); assert.equal(cancels('omacom/omarchy-pkgs', 'auto/sync-rebuilds'), false); + assert.equal(cancels('omacom/omarchy-pkgs', 'auto/track-branches-omarchy-dev-omarchy-settings-dev'), false); assert.equal(cancels('omacom/omarchy-pkgs', 'ttfx/fix'), true); assert.equal(cancels('someone/omarchy-pkgs', 'auto/sync-upstream'), true); assert.equal(cancels(undefined, ''), true); // workflow_dispatch diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index 65dfd9a4..b40616af 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -250,14 +250,41 @@ b2sums=('old' 'local-b2') with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'): w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P[0-9.]+)', self.root / 'other-cache') - def test_git_branch_min_age_holds_the_tip_instead_of_selecting_history(self): + def test_git_branch_min_age_falls_back_to_the_newest_old_enough_tip(self): repo, shas = self.branch_fixture(fresh_tip=True) watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} with self.redirect_clone(repo): releases = w.discover(watch, self.fetch) + held = w.discover(watch, self.fetch, min_age=3600) self.assertEqual(w.select_release(releases)['values']['commit'], shas[-1]) + # The backstop still refuses the fresh tip itself... self.assertIsNone(w.select_release(releases, min_age=3600)) self.assertEqual(w.select_release(releases, min_age=3600, bypass=True)['values']['commit'], shas[-1]) + # ...and the watch offers the commit before it instead of nothing. + self.assertEqual(w.select_release(held, min_age=3600)['values']['commit'], shas[-2]) + self.assertEqual(held[0]['values']['count'], str(len(shas) - 1)) + + def test_git_branch_min_age_never_falls_back_into_a_merged_side_branch(self): + repo, shas = self.branch_fixture() + git = ['git', '-C', str(repo), '-c', 'user.name=Test', '-c', 'user.email=test@example.test'] + old = {**os.environ, 'GIT_COMMITTER_DATE': '2020-01-02T00:00:00+00:00', 'GIT_AUTHOR_DATE': '2020-01-02T00:00:00+00:00'} + subprocess.run([*git, 'checkout', '-q', '-b', 'side'], check=True) + (repo / 'side').write_text('x') + subprocess.run([*git, 'add', '.'], check=True) + subprocess.run([*git, 'commit', '-qm', 'side'], env=old, check=True) + subprocess.run([*git, 'checkout', '-q', 'main'], check=True) + subprocess.run([*git, 'merge', '-q', '--no-ff', '-m', 'merge side', 'side'], check=True) + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + with self.redirect_clone(repo): + held = w.discover(watch, self.fetch, min_age=3600) + self.assertEqual(w.select_release(held, min_age=3600)['values']['commit'], shas[-1]) + + def test_git_branch_min_age_holds_when_nothing_is_old_enough(self): + repo, shas = self.branch_fixture(fresh_tip=True) + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + with self.redirect_clone(repo): + held = w.discover(watch, self.fetch, min_age=10 ** 10) + self.assertIsNone(w.select_release(held, min_age=10 ** 10)) def branch_sync_fixture(self): repo, shas = self.branch_fixture()