From d87686ca4f86fea3bf8717a42cc1e4a8462ec1f2 Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Mon, 21 Sep 2026 03:05:03 -0400 Subject: [PATCH 1/4] Track upstream branches as pinned releases on an unattended lane Since publishing moved to CI on merge, a package whose PKGBUILD never changes while its source moves was never rebuilt: omarchy-dev and omarchy-settings-dev followed quattro through "#branch=" and a pkgver() function, and nothing in this repository changed when quattro did. The host timers that used to notice are off, so edge fell days behind. The rule now: no git source without a commit or tag pin (tests/pinned-sources.sh, run in CI). A package that has to follow a branch declares a git_branch upstream watch, and the pin moves through the same PR/build/publish path as every other version bump. Watch (helpers/upstream-watch.py) git_branch gains tag_pattern: the newest release tag in the pinned commit's own history, exposed as {tag}/{version}/{distance}, so a branch build is versioned .r.g, above the release it follows and below the next one. One blobless clone per branch per run, shared by every package on it. min_release_age selects the newest commit older than the window, so a push burst builds once. Lane (helpers/package-metadata.sh, bin/sync-upstream --lane) "auto_merge": true moves a package from the reviewed 6-hourly sync PR to the unattended lane. Packages pinned from the same branch move together: a failure on one restores the others and fails the group, so the dev pair can never ship from two quattro commits. Tracker (.github/workflows/track-branches.yml) Every two hours: pin, open one PR with a GitHub App token, enable auto-merge. Branch protection still gates the merge on result, self-tests and build-isolation. A tip that fails to build stays an open red PR until the next tick supersedes it. The App is required: a PR opened with GITHUB_TOKEN has its checks held for approval and its auto-merge would not fire publish.yml. The reviewed workflows (sync-upstream, sync-rebuilds) open their PRs with the same App so their builds start without a maintainer clicking "Approve workflows to run"; without the App they fall back to GITHUB_TOKEN and behave as before. Recipes The dev pair pins _commit and a real sha256sum, keeps the OMARCHY_SRC override, and drops pkgver(). Its r-number stays the branch's total commit count because the published history used it and pacman must never see the version go down. omasnap-git is new: omacom/omasnap main, versioned .r.g, provides/conflicts omasnap. --- .github/workflows/sync-rebuilds.yml | 20 +- .github/workflows/sync-upstream.yml | 29 ++- .github/workflows/test.yml | 1 + .github/workflows/track-branches.yml | 177 ++++++++++++++++++ README.md | 20 +- bin/sync-upstream | 87 ++++++++- docs/upstream-sources.md | 38 +++- helpers/package-metadata.sh | 35 ++++ helpers/upstream-watch.py | 84 ++++++++- pkgbuilds/omarchy-dev/.omarchy/package.json | 18 +- pkgbuilds/omarchy-dev/PKGBUILD | 34 ++-- .../.omarchy/package.json | 18 +- pkgbuilds/omarchy-settings-dev/PKGBUILD | 36 ++-- pkgbuilds/omasnap-git/.omarchy/package.json | 17 ++ pkgbuilds/omasnap-git/PKGBUILD | 72 +++++++ tests/pinned-sources.sh | 49 +++++ tests/upstream-watch.py | 79 ++++++++ 17 files changed, 746 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/track-branches.yml create mode 100644 pkgbuilds/omasnap-git/.omarchy/package.json create mode 100644 pkgbuilds/omasnap-git/PKGBUILD create mode 100755 tests/pinned-sources.sh diff --git a/.github/workflows/sync-rebuilds.yml b/.github/workflows/sync-rebuilds.yml index 4d8af048..a921e83b 100644 --- a/.github/workflows/sync-rebuilds.yml +++ b/.github/workflows/sync-rebuilds.yml @@ -68,11 +68,27 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi + # App token rather than GITHUB_TOKEN so the PR's build and test runs + # start without a maintainer approving them (see sync-upstream.yml). + - name: Mint the bot token + if: steps.changes.outputs.has_changes == 'true' + id: app + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + # Without the App configured this falls back to GITHUB_TOKEN below, + # which still opens the PR; a maintainer then has to approve its + # workflow runs by hand, as before. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.PKGS_BOT_APP_ID }} + private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + continue-on-error: true + - name: Create Pull Request if: steps.changes.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} commit-message: 'chore: rebuild against updated dependencies' title: 'chore: rebuild against updated dependencies' body: | diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 2e691367..971d461b 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -47,11 +47,13 @@ jobs: useradd -m -u "$HOST_UID" -g "$HOST_GID" runner chown -R runner:runner /workspace/pkgbuilds + # The reviewed lane only: packages marked auto_merge ride + # track-branches.yml, which merges without a human. if [[ -n "${PACKAGES:-}" ]]; then read -r -a package_args <<< "$PACKAGES" - runuser -u runner -- ./bin/sync-upstream "${package_args[@]}" + runuser -u runner -- ./bin/sync-upstream --lane reviewed "${package_args[@]}" else - runuser -u runner -- ./bin/sync-upstream + runuser -u runner -- ./bin/sync-upstream --lane reviewed fi ' env: @@ -70,11 +72,30 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi + # A PR opened with GITHUB_TOKEN gets its build and test runs held + # until a maintainer clicks "Approve workflows to run"; one opened by + # the App builds on its own, so the reviewer sees a green (or red) PR + # instead of a pending one. The App only opens the PR: merging stays + # a human decision in this lane. + - name: Mint the bot token + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: app + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + # Without the App configured this falls back to GITHUB_TOKEN below, + # which still opens the PR; a maintainer then has to approve its + # workflow runs by hand, as before. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.PKGS_BOT_APP_ID }} + private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + continue-on-error: true + - name: Create Pull Request if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} commit-message: 'chore: sync upstream releases' title: 'chore: sync upstream releases' body: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bba4b01e..1c300687 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,6 +58,7 @@ jobs: python tests/neovim-clipboard-tmux.py ./tests/partial-release.sh ./tests/published-build-plan.sh + ./tests/pinned-sources.sh ./tests/controller.sh ./tests/artifact-helpers.sh ./tests/limine-mkinitcpio-hook.sh diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml new file mode 100644 index 00000000..5413855a --- /dev/null +++ b/.github/workflows/track-branches.yml @@ -0,0 +1,177 @@ +name: Track upstream branches + +# The unattended lane. Packages marked "auto_merge": true follow a moving +# upstream branch (omarchy-dev and omarchy-settings-dev on quattro, omasnap-git +# on main) rather than tagged releases, so nothing in this repository changes +# when their source does. This workflow makes each new branch tip a commit pin +# in the recipe, which publish.yml then treats like any other version bump: +# the PR builds on the droplets, auto-merge lands it when `result` is green, +# and the merge publishes the artifacts. A tip that fails to build stays an +# unmerged red PR that the next tick supersedes. +# +# The PR is opened with a GitHub App token, not GITHUB_TOKEN: a pull request +# created with the workflow token gets its CI runs held for manual approval, +# and an auto-merge it enabled would not fire the publish workflow. The App +# needs Contents: write and Pull requests: write on this repository; its id +# and private key live in the PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY secrets. + +on: + schedule: + # Every 2 hours, off the hour to dodge the scheduling backlog at :00 + - cron: '35 */2 * * *' + workflow_dispatch: + inputs: + packages: + description: 'Specific packages to track (space-separated, leave empty for every auto_merge package)' + required: false + default: '' + +# One tracker at a time: two runs racing on auto/track-branches would each +# force-push their own pin over the other's. +concurrency: + group: track-branches + cancel-in-progress: false + +jobs: + track: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + # Same container as the reviewed sync: vercmp decides whether a pin is + # an upgrade with the comparator pacman uses on users' machines. + - name: Pin tracked branches to their current tips + id: sync + run: | + docker run --rm \ + -e PACKAGES="$PACKAGES" \ + -e UPSTREAM_GITHUB_TOKEN="$UPSTREAM_GITHUB_TOKEN" \ + -e HOST_UID="$(id -u)" \ + -e HOST_GID="$(id -g)" \ + -v "$PWD/bin:/workspace/bin:ro" \ + -v "$PWD/helpers:/workspace/helpers:ro" \ + -v "$PWD/pkgbuilds:/workspace/pkgbuilds" \ + -w /workspace \ + archlinux:base-devel bash -lc ' + set -euo pipefail + + pacman -Syu --noconfirm git jq python libarchive + + groupadd -g "$HOST_GID" runner + useradd -m -u "$HOST_UID" -g "$HOST_GID" runner + chown -R runner:runner /workspace/pkgbuilds + + if [[ -n "${PACKAGES:-}" ]]; then + read -r -a package_args <<< "$PACKAGES" + runuser -u runner -- ./bin/sync-upstream --lane auto-merge "${package_args[@]}" + else + runuser -u runner -- ./bin/sync-upstream --lane auto-merge + fi + ' + env: + PACKAGES: ${{ github.event.inputs.packages }} + UPSTREAM_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for changes + if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }} + id: changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "has_changes=true" >> "$GITHUB_OUTPUT" + git status --porcelain + { + echo "### Pinned" + git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' | grep -E '^\+(pkgver|_commit)=' | sed 's/^+/- /' + } >> "$GITHUB_STEP_SUMMARY" + fi + + # No fallback to GITHUB_TOKEN here: a PR it opened would sit with its + # checks held, and an auto-merge it enabled would land without running + # publish.yml. Better to fail loudly than to pin quietly. + - name: Require the bot App + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + run: | + if [[ -z "$PKGS_BOT_APP_ID" ]]; then + echo "::error::PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY are not set. Create a GitHub App with Contents: write and Pull requests: write, install it on this repository, and store its id and private key as those secrets." + exit 1 + fi + + - name: Mint the bot token + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: app + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.PKGS_BOT_APP_ID }} + private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + + # The PR title names what moved, so the merged history reads like a + # changelog: "Track upstream branches: omarchy-dev 4.0.0.r6520.g1a2b3c4, ...". + - name: Describe the pins + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: describe + run: | + title=$(git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' \ + | awk '/^\+\+\+ b\/pkgbuilds\//{split($2,p,"/"); pkg=p[3]} /^\+pkgver=/{sub(/^\+pkgver=/,""); printf "%s %s, ", pkg, $0}' \ + | sed 's/, $//') + echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" + + - name: Open or update the tracking PR + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: pr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.app.outputs.token }} + commit-message: ${{ steps.describe.outputs.title }} + title: ${{ steps.describe.outputs.title }} + body: | + Automated pin of packages that follow a moving upstream branch + (`"auto_merge": true` in `.omarchy/package.json`). Each package's + `_commit` now points at the branch tip that has been there for at + least its `min_release_age`. + + This PR auto-merges once the build checks pass. A failing build + leaves it open; the next tracker run replaces it with the newer tip. + branch: auto/track-branches + delete-branch: true + labels: automated + + # Auto-merge, not a direct merge: branch protection still has to see + # `result`, `self-tests` and `build-isolation` green, and this lane + # inherits every rule the reviewed lane has except the human. + - name: Enable auto-merge + if: ${{ !cancelled() && steps.pr.outputs.pull-request-number != '' }} + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + PR: ${{ steps.pr.outputs.pull-request-number }} + run: | + # Idempotent across re-runs of an updated PR: enabling twice errors. + if [[ "$(gh pr view "$PR" -R "${{ github.repository }}" --json autoMergeRequest --jq '.autoMergeRequest != null')" == true ]]; then + echo "auto-merge already enabled on #$PR" + exit 0 + fi + # A PR whose checks all reused existing artifacts can be clean + # before this step runs; GitHub then refuses --auto, so merge it. + gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" \ + || gh pr merge --merge "$PR" -R "${{ github.repository }}" + + - name: Notify Basecamp on failure + if: failure() && env.BASECAMP_CHATBOT_URL != '' + env: + BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} + run: | + curl -s -o /dev/null \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg content \ + "🔴 Branch tracking failed
View run" \ + '{content: $content}')" \ + "$BASECAMP_CHATBOT_URL" diff --git a/README.md b/README.md index 8823595e..5e216f7f 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,14 @@ The filesystem no longer encodes release policy. Instead: (`OMARCHY_RC_PINS=1`, which `omarchy-release rc` sets) may build it for rc — master's shipped pins can never overwrite an in-flight RC. The dev pair (`omarchy-dev`, `omarchy-settings-dev`) is pinned to `edge` +- packages that follow a moving upstream branch (the dev pair on `quattro`, + `omasnap-git` on `main`) still pin an exact commit in their PKGBUILD. A + `git_branch` upstream watch moves that pin, and `"auto_merge": true` puts the + package on the unattended lane: `track-branches.yml` opens the bump PR every + two hours and auto-merges it once the build checks pass, so a branch tip + reaches the edge channel without anyone clicking. No PKGBUILD may carry an + unpinned git source (`tests/pinned-sources.sh`); a branch that has to be + followed gets a watch, not a `#branch=` fragment - Omarchy owns every checked-in recipe; upstream watches update release metadata without replacing packaging or architecture support - packages can opt out of unscoped builds with `skip_build`; explicit `--package` builds remain available - packages follow direct upstream watches/providers in `.omarchy/package.json`, or a custom `.omarchy/upstream.sh` hook @@ -719,6 +727,7 @@ Fields: - `upstream`: optional direct release watch (see [Upstream watches](docs/upstream-sources.md)), or an existing GitHub, git-tag, npm, or Debian provider. GitHub architecture assets may be a string or an ordered array, and can be combined with disjoint versioned `sources` — see [Sync Upstream Releases](#sync-upstream-releases). Mutually exclusive with `.omarchy/upstream.sh`. - `min_release_age`: optional quarantine for upstream releases (`"24h"`, `"2d"`, or bare seconds). The newest release older than the window ships; anything younger waits, and a release whose age cannot be proven fails the sync. Bypass deliberately with `BYPASS_MIN_RELEASE_AGE=1 bin/sync-upstream `. - `sync`: `false` records an existing manual maintenance hold. Held packages have no upstream watch/provider/hook and are excluded from automatic updates. +- `auto_merge`: optional boolean; defaults to `false`. `true` moves the package's upstream updates from the reviewed 6-hourly sync PR to the unattended lane: `track-branches.yml` opens its bump PR and auto-merges it when CI is green. Meant for packages that follow a moving branch through a `git_branch` watch, where every tip is a release and there is nothing for a reviewer to read. Requires an upstream watch, provider, or hook. - `origin`: optional historical import provenance, with `aur` (package name) and `commit`. It does not control updates. - `release_ring`: optional. `fast` means the package is built directly for stable as well as edge, with the artifacts replicated into rc for parity. Packages without a ring build in edge and reach stable through the pipeline (`bin/repo advance`). - `channels`: optional array bounding where the package may be built (`edge`, `rc`, `stable`). Without the key a package is a member of every channel and follows the default build rules above; `bin/repo advance` refuses to carry a package anywhere it isn't a member. @@ -880,8 +889,17 @@ The repository includes GitHub workflows and systemd services for automated rele #### GitHub Workflows -1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red. +1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes on the reviewed lane. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red. 2. **sync-rebuilds.yml** (Every 6 hours): Bumps pkgrel for packages whose `rebuild_on` dependencies have moved in the official repositories and opens a PR. +3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the newest tip of its watched branch that has sat there for `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. + +The sync PRs are opened with a GitHub App token (`PKGS_BOT_APP_ID` and +`PKGS_BOT_PRIVATE_KEY` secrets; the App needs Contents and Pull requests +write on this repository). A PR opened with the workflow's own `GITHUB_TOKEN` +has its build and test runs held until a maintainer approves them, and an +auto-merge it enabled would land without running the publish workflow. The +reviewed workflows fall back to `GITHUB_TOKEN` when the App is not configured +(and then need that click); the tracker refuses to run without it. To approve builds for an unvouched contributor's PR, apply **`build-approved`**. Until approval, the PR shows **Awaiting build approval** and its required diff --git a/bin/sync-upstream b/bin/sync-upstream index 2e22f9ae..a08f184d 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -12,6 +12,7 @@ trap 'rm -rf "$TEMP_DIR"' EXIT export UPSTREAM_CACHE_DIR="$TEMP_DIR/watch-cache" SPECIFIC_PACKAGES=() +LANE=all usage() { cat < + Which delivery lane to sync (default: all). Packages marked + "auto_merge": true ride the unattended lane (the branch tracker + opens and auto-merges their PR); everything else is reviewed. + The scheduled workflows each pass their own lane so a moving + branch tip never waits on a vendor release, or the reverse. + Commands: self-test Run the offline fixture tests for release selection, the quarantine backstop, and metadata parsing @@ -65,6 +74,14 @@ while [[ $# -gt 0 ]]; do usage exit 0 ;; + --lane) + LANE="${2:-}" + case "$LANE" in + reviewed|auto-merge|all) ;; + *) print_error "Invalid --lane '$LANE' (expected reviewed, auto-merge, or all)"; exit 1 ;; + esac + shift 2 + ;; --*) print_error "Unknown option: $1" exit 1 @@ -1042,16 +1059,82 @@ if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 && "${SPECIFIC_PACKAGES[0]}" == "self-test" exit 0 fi +# Packages that pin the same upstream branch move together or not at all. +# The watch reads one shared clone per run, so they only disagree when one +# package's update failed after the tip was chosen (a checksum fetch, say). +# Leaving the other one advanced would ship omarchy-dev and +# omarchy-settings-dev from different commits, which is exactly the skew the +# release pair's lockstep guard exists to prevent. Restore the packages that +# moved and count the group as failed; the next run tries again. +declare -A BEFORE_SYNC=() + +snapshot_package() { + local package="$1" pkgbuild="$PKGBUILDS_DIR/$1/PKGBUILD" + [[ -f "$pkgbuild" ]] || return 0 + mkdir -p "$TEMP_DIR/before" + cp "$pkgbuild" "$TEMP_DIR/before/$package" + BEFORE_SYNC["$package"]=1 +} + +branch_watch_key() { + local package_dir="$1" + jq -r ' + (.upstream.watch? | objects | select(has("git_branch"))) + | "\(.git_branch)#\(.branch)" + ' "$package_dir/.omarchy/package.json" 2>/dev/null +} + +enforce_branch_lockstep() { + local package key + declare -A groups=() + for package in "${!BEFORE_SYNC[@]}"; do + key=$(branch_watch_key "$PKGBUILDS_DIR/$package") + [[ -n "$key" ]] || continue + groups["$key"]+="$package " + done + for key in "${!groups[@]}"; do + local members commits + read -r -a members <<<"${groups[$key]}" + (( ${#members[@]} > 1 )) || continue + commits=$(for package in "${members[@]}"; do + grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" + done | sort -u | grep -c .) + (( commits > 1 )) || continue + print_error "Lockstep violation on $key: ${members[*]} pin different commits; restoring all of them" + for package in "${members[@]}"; do + if ! cmp -s "$TEMP_DIR/before/$package" "$PKGBUILDS_DIR/$package/PKGBUILD"; then + cp "$TEMP_DIR/before/$package" "$PKGBUILDS_DIR/$package/PKGBUILD" + # Not ((--UPDATED)): an arithmetic command that evaluates to zero + # returns 1, and under errexit that would end the run right here. + UPDATED=$((UPDATED > 0 ? UPDATED - 1 : 0)) + fi + done + ((++FAILED)) + done +} + +sync_in_lane() { + local package="$1" package_dir="$PKGBUILDS_DIR/$1" + if [[ -d "$package_dir" ]] && ! package_in_lane "$package_dir" "$LANE"; then + print_info "Skipping $package: not in the $LANE lane" + ((++SKIPPED)) + return 0 + fi + snapshot_package "$package" + sync_package "$package" +} + if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then SPECIFIC_MODE=true for package in "${SPECIFIC_PACKAGES[@]}"; do - sync_package "$package" + sync_in_lane "$package" done else while IFS= read -r package; do - sync_package "$package" + sync_in_lane "$package" done < <(packages_for_upstream_sync) fi +enforce_branch_lockstep echo "" if [[ $FAILED -gt 0 ]]; then diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index 9ef208f1..79cff76b 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -48,8 +48,40 @@ pkgrel to 1, but the complete epoch:pkgver-pkgrel must still increase. GitHub releases exclude drafts and prereleases unless `allow_prerelease` is true. Existing `min_release_age` policies apply: a feed without a verifiable publication -time cannot bypass a configured hold. Git branch watches derive a commit count -and date from the actual branch history and write an immutable source pin. +time cannot bypass a configured hold. + +## Branch watches + +A `git_branch` watch treats every commit on a branch as a release and writes an +immutable pin (`"_commit": "{commit}"`) so the recipe never carries a moving +`#branch=` source; `tests/pinned-sources.sh` enforces that. The clone is bare, +blobless and single-branch, read only with git, and shared by every package +that watches the same branch in one run, so two recipes pinned from it always +see the same commit. Values available to `version`: + +- `{date}` (default), `{count}` (commits on the branch), `{commit}` + (`{commit:.7}` for the short form) +- with `tag_pattern` (a regular expression with a named `version` group, + matched against whole tags): `{tag}`, `{version}` from that tag, and + `{distance}`, the number of commits past it. Only tags in the pinned + commit's own history count, so a release cut on another branch is ignored. + +`{version}.r{distance}.g{commit:.7}` gives `1.21.0.r15.gabc1234`, which pacman +orders above the `1.21.0` release it follows and below `1.21.1`; `omasnap-git` +uses it. The Omarchy dev pair uses `{version}.r{count}.g{commit:.7}` instead +because its published history counted every commit and the number must never +go down. + +`min_release_age` on a branch watch selects the newest commit that has been on +the branch for at least that long, so a burst of pushes builds once after it +settles rather than once per push. `BYPASS_MIN_RELEASE_AGE=1` takes the tip. + +Packages marked `"auto_merge": true` ride the unattended lane +(`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened +and auto-merged as soon as the build checks pass. `bin/sync-upstream --lane +reviewed|auto-merge|all` selects a lane; the scheduled workflows each pass their +own. Packages that pin the same branch move in lockstep: if one of them fails +to update, the run restores the others and reports the group as failed. Checksums retain their algorithms (SHA256, SHA512, BLAKE2, etc.) and source order. Changed git sources are hashed with makepkg's git-archive convention. Unchanged @@ -131,6 +163,8 @@ in `origin` and has no effect on release selection. | `localsend` | github | [localsend/localsend](https://github.com/localsend/localsend) | | `localsend-bin` | github | [localsend/localsend](https://github.com/localsend/localsend) | | `macbook12-spi-driver-dkms` | git_branch | [https://github.com/marc-git/macbook12-spi-driver.git](https://github.com/marc-git/macbook12-spi-driver.git) | +| `omarchy-dev`, `omarchy-settings-dev` | git_branch (auto-merge) | [https://github.com/basecamp/omarchy.git](https://github.com/basecamp/omarchy.git) `quattro` | +| `omasnap-git` | git_branch (auto-merge) | [https://github.com/omacom/omasnap.git](https://github.com/omacom/omasnap.git) `main` | | `makima-bin` | github | [cyber-sushi/makima](https://github.com/cyber-sushi/makima) | | `minecraft-launcher` | archive | [https://launcher.mojang.com/download/Minecraft.deb](https://launcher.mojang.com/download/Minecraft.deb) | | `nautilus-dropbox` | github | [dropbox/nautilus-dropbox](https://github.com/dropbox/nautilus-dropbox) | diff --git a/helpers/package-metadata.sh b/helpers/package-metadata.sh index ce46e921..23288bc3 100644 --- a/helpers/package-metadata.sh +++ b/helpers/package-metadata.sh @@ -13,6 +13,7 @@ # { "source": "local", "channels": ["edge"] } # { "source": "local", "channels": ["edge", "rc", "stable"] } # { "source": "local", "min_release_age": "24h" } +# { "source": "local", "auto_merge": true, "upstream": { "watch": { "git_branch": "...", "branch": "main" } } } # { "source": "local", "upstream": { "github": "owner/repo", "checksums": "SHASUMS256.txt", "assets": { "x86_64": ["name-{tag}-x64.tar.xz"] } } } # { "source": "local", "upstream": { "github": "owner/repo", "digests": true, "assets": { "x86_64": "name-{tag}-x64.tar.xz" } } } # { "source": "local", "upstream": { "git_tags": "https://example/repo.git", "tag_pattern": "v{pkgver}", "sources": { "any": ["https://example/archive/{tag}.tar.gz"] } } } @@ -327,6 +328,31 @@ packages_for_upstream_sync() { done } +# Upstream updates travel in one of two lanes. The reviewed lane is the +# 6-hourly sync PR a maintainer reads before merging. A package that marks +# "auto_merge": true rides the unattended lane instead: its bump PR is opened +# and auto-merged by the branch tracker as soon as CI is green, which is how a +# package that follows a moving branch (omarchy-dev, omasnap-git) gets rebuilt +# without anyone clicking. The lanes are disjoint so a branch tip can never +# hold up a reviewed vendor release, or the other way round. +package_auto_merge() { + local pkgdir="$1" metadata + metadata=$(metadata_file_for_dir "$pkgdir") + [[ -f "$metadata" ]] || return 1 + [[ "$(jq -r 'if has("auto_merge") then .auto_merge else false end' "$metadata")" == "true" ]] +} + +# package_in_lane +package_in_lane() { + local pkgdir="$1" lane="$2" + case "$lane" in + all | "") return 0 ;; + auto-merge) package_auto_merge "$pkgdir" ;; + reviewed) ! package_auto_merge "$pkgdir" ;; + *) echo "invalid lane: $lane (expected reviewed, auto-merge, or all)" >&2; return 2 ;; + esac +} + # Packages that must be rebuilt when a dependency they link against changes, # even though nothing in their own source moved. `rebuild_on` names those # dependencies; `rebuilt_against` records the versions the checked-in pkgrel was @@ -514,6 +540,15 @@ validate_package_metadata() { return 1 fi + if ! jq -e 'if has("auto_merge") | not then true else (.auto_merge | type) == "boolean" end' "$metadata" >/dev/null; then + echo "invalid auto_merge for $(basename "$pkgdir"): must be boolean" + return 1 + fi + if package_auto_merge "$pkgdir" && ! package_has_upstream_provider "$pkgdir" && ! package_has_upstream_hook "$pkgdir"; then + echo "invalid auto_merge for $(basename "$pkgdir"): only an upstream watch, provider, or hook can be auto-merged" + return 1 + fi + # `has` rather than `// {}`: jq's // treats false as absent, which would # let "upstream": false slip through as an empty declaration. if ! jq -e ' diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index 6e17b979..05828f4b 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -82,7 +82,7 @@ def validate(watch): allowed = PROVIDERS | {"pattern", "path", "package", "branch", "variables", "fields", "submodules", "allow_prerelease", "unescape_json", "filenames", "sequence", "version", "revision", "revision_variable", - "mutable_sources", "member", "dist_tag"} + "mutable_sources", "member", "dist_tag", "tag_pattern"} if watch.keys() - allowed: raise ValueError(f"unknown watch fields: {sorted(watch.keys() - allowed)}") value = watch[provider] @@ -113,6 +113,18 @@ def validate(watch): if not isinstance(branch, str) or not branch or branch.startswith("-"): raise ValueError("git branch watch needs an explicit branch") run(["git", "check-ref-format", "refs/heads/" + branch]) + # A branch watch whose version template names a tag needs to know + # which tags count as releases; anything else is an untagged branch. + if "tag_pattern" in watch: + if not isinstance(watch["tag_pattern"], str) or not watch["tag_pattern"]: + raise ValueError("watch.tag_pattern must be a regular expression string") + if "version" not in re.compile(watch["tag_pattern"]).groupindex: + raise ValueError("tag_pattern needs a named version group") + template = watch.get("version", "{version}") + if any(field in template for field in ("{tag", "{distance")) and "tag_pattern" not in watch: + raise ValueError("a version built from {tag}/{distance} needs a tag_pattern") + elif "tag_pattern" in watch: + raise ValueError("tag_pattern only applies to git_branch watches") for field in ("variables", "submodules", "fields"): mapping = watch.get(field, {}) if not isinstance(mapping, dict): @@ -172,7 +184,61 @@ def matches(watch, text, extra=None, full=False): yield candidate(watch, {**(extra or {}), **match.groupdict()}) -def discover(watch, fetch): +def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): + """Describe the newest commit on an upstream branch that has sat there for + at least min_age seconds (the branch analogue of "the newest release older + than the window ships"): commit, total count, date, and with a tag_pattern + the newest release tag reachable from it plus the distance from that tag, + so a branch build can be versioned .r.g, above the release it + follows and below the next one, the way a pkgver() function would. + + One blobless single-branch clone per (url, branch) per run, shared by + every package that tracks it, so two recipes pinned from one clone always + see the same commit. The clone is read with git only; nothing in it runs. + Returns None when every commit is younger than the window. + """ + https(url) + key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest() + work = Path(cache) / f"{key}.branch.git" + if not work.exists(): + scratch = work.with_name(f"{work.name}.{os.getpid()}.tmp") + subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True) + scratch.replace(work) + git = ["git", "-C", str(work)] + selector = ["HEAD"] + if min_age: + cutoff = (now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(seconds=min_age) + selector = ["-1", f"--before={cutoff.isoformat()}", "HEAD"] + commit = run([*git, "rev-list", *selector], text=True).split()[:1] + if not commit: + return None + commit = commit[0] + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("branch tip is not a commit") + count = run([*git, "rev-list", "--count", commit], text=True).strip() + date = run([*git, "show", "-s", "--format=%cs", commit], text=True).strip().replace("-", "") + timestamp = run([*git, "show", "-s", "--format=%cI", commit], text=True).strip() + values = {"version": date, "date": date, "count": count, "commit": commit, "published_at": timestamp} + if tag_pattern: + pattern = re.compile(tag_pattern) + best = None + # Only tags in this commit's history count; a release cut on another + # branch is not something this branch is "past". + for tag in run([*git, "tag", "--merged", commit], text=True).split(): + match = pattern.fullmatch(tag) + if not match: + continue + version = match.group("version") + if best is None or vercmp(version, best[0]) > 0: + best = (version, tag) + if best is None: + raise ValueError(f"no tag on {branch} matches {tag_pattern}") + distance = run([*git, "rev-list", "--count", f"{best[1]}..{commit}"], text=True).strip() + values.update({"tag": best[1], "version": best[0], "distance": distance}) + return values + + +def discover(watch, fetch, min_age=0): provider = validate(watch) feed = watch[provider] results = [] @@ -199,13 +265,10 @@ def discover(watch, fetch): for tag, commit in tags.items(): results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True)) elif provider == "git_branch": - with tempfile.TemporaryDirectory(prefix="upstream-git-") as work: - subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", watch["branch"], feed, work], check=True) - commit = run(["git", "-C", work, "rev-parse", "HEAD"], text=True).strip() - count = run(["git", "-C", work, "rev-list", "--count", "HEAD"], text=True).strip() - date = run(["git", "-C", work, "show", "-s", "--format=%cs", "HEAD"], text=True).strip().replace("-", "") - timestamp = run(["git", "-C", work, "show", "-s", "--format=%cI", "HEAD"], text=True).strip() - results.append(candidate(watch, {"version": date, "date": date, "count": count, "commit": commit, "published_at": timestamp})) + tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age) + if tip is None: + return [] # nothing has settled for min_age yet: wait, not an error + results.append(candidate(watch, tip)) elif provider == "npm": data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe="")) version = data["dist-tags"][watch.get("dist_tag", "latest")] @@ -479,7 +542,8 @@ def sync(package, fetch, min_age=0, check=False): path = package / "PKGBUILD" original = path.read_text() before = read_recipe(path) - release = select_release(discover(watch, fetch), min_age, bypass=os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1") + bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1" + release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass) if release is None: return {"status": "skipped", "reason": "minimum release age"} current = scalar(before, "pkgver") diff --git a/pkgbuilds/omarchy-dev/.omarchy/package.json b/pkgbuilds/omarchy-dev/.omarchy/package.json index 9adf3728..64dc2057 100644 --- a/pkgbuilds/omarchy-dev/.omarchy/package.json +++ b/pkgbuilds/omarchy-dev/.omarchy/package.json @@ -1 +1,17 @@ -{ "source": "local", "channels": ["edge"] } +{ + "source": "local", + "channels": ["edge"], + "auto_merge": true, + "min_release_age": "30m", + "upstream": { + "watch": { + "git_branch": "https://github.com/basecamp/omarchy.git", + "branch": "quattro", + "tag_pattern": "v(?P[0-9]+\\.[0-9]+\\.[0-9]+)", + "version": "{version}.r{count}.g{commit:.7}", + "variables": { + "_commit": "{commit}" + } + } + } +} diff --git a/pkgbuilds/omarchy-dev/PKGBUILD b/pkgbuilds/omarchy-dev/PKGBUILD index 58630edd..c2e70eac 100644 --- a/pkgbuilds/omarchy-dev/PKGBUILD +++ b/pkgbuilds/omarchy-dev/PKGBUILD @@ -1,9 +1,13 @@ # Maintainer: Ryan Hughes pkgname='omarchy-dev' -pkgver=4.0.0.r847.g4de185b +pkgver=4.0.0.r6514.gee8ebf6 pkgrel=1 -_pkgver_base=4.0.0 -_pkgver_base_tag=v3.8.2 +# Pinned by the upstream watch in .omarchy/package.json (bin/sync-upstream): +# every quattro tip becomes a commit pin here, so the package is versioned, +# checksummed and built exactly like a release, just more often. The r-number +# is the branch's total commit count, not the distance from the last tag: the +# published history used the total, and pacman must never see it go down. +_commit=ee8ebf615dcbf9e1a126b22a894f907d6795a9a5 pkgdesc='Beautiful, modern, and opinionated Arch Linux by DHH (quattro branch tip)' # The payload is architecture-independent, but the dependency set is not: the # boot stack differs per architecture (see depends_x86_64 / depends_aarch64), @@ -73,30 +77,16 @@ makedepends=( 'git' ) -# Source: quattro branch tip by default; set OMARCHY_SRC=/path/to/checkout -# to build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX). +# Source: the pinned quattro commit. Set OMARCHY_SRC=/path/to/checkout to +# build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX); +# the arrays are emptied below so nothing is downloaded in that case. +source=("omarchy::git+https://github.com/basecamp/omarchy.git#commit=${_commit}") +sha256sums=('4f55a96e32143c766f925ec7f721a374c32bd20e66f9a081f5dbe0772c609ecd') if [[ -n "${OMARCHY_SRC:-}" ]]; then source=() sha256sums=() -else - source=("omarchy::git+https://github.com/basecamp/omarchy.git#branch=quattro") - sha256sums=('SKIP') fi -pkgver() { - cd "$srcdir/omarchy" - - local commit_count commit_hash - if git rev-parse --verify "${_pkgver_base_tag}^{commit}" >/dev/null 2>&1; then - commit_count=$(git rev-list --count "${_pkgver_base_tag}..HEAD") - else - commit_count=$(git rev-list --count HEAD) - fi - commit_hash=$(git rev-parse --short=7 HEAD) - - printf "%s.r%s.g%s" "$_pkgver_base" "$commit_count" "$commit_hash" -} - prepare() { if [[ -n "${OMARCHY_SRC:-}" ]]; then rm -rf "$srcdir/omarchy" diff --git a/pkgbuilds/omarchy-settings-dev/.omarchy/package.json b/pkgbuilds/omarchy-settings-dev/.omarchy/package.json index 9adf3728..64dc2057 100644 --- a/pkgbuilds/omarchy-settings-dev/.omarchy/package.json +++ b/pkgbuilds/omarchy-settings-dev/.omarchy/package.json @@ -1 +1,17 @@ -{ "source": "local", "channels": ["edge"] } +{ + "source": "local", + "channels": ["edge"], + "auto_merge": true, + "min_release_age": "30m", + "upstream": { + "watch": { + "git_branch": "https://github.com/basecamp/omarchy.git", + "branch": "quattro", + "tag_pattern": "v(?P[0-9]+\\.[0-9]+\\.[0-9]+)", + "version": "{version}.r{count}.g{commit:.7}", + "variables": { + "_commit": "{commit}" + } + } + } +} diff --git a/pkgbuilds/omarchy-settings-dev/PKGBUILD b/pkgbuilds/omarchy-settings-dev/PKGBUILD index 1d4dce04..2fb10166 100644 --- a/pkgbuilds/omarchy-settings-dev/PKGBUILD +++ b/pkgbuilds/omarchy-settings-dev/PKGBUILD @@ -1,9 +1,13 @@ # Maintainer: Ryan Hughes pkgname='omarchy-settings-dev' -pkgver=4.0.0.r847.g4de185b -pkgrel=2 -_pkgver_base=4.0.0 -_pkgver_base_tag=v3.8.2 +pkgver=4.0.0.r6514.gee8ebf6 +pkgrel=1 +# Pinned by the upstream watch in .omarchy/package.json (bin/sync-upstream): +# every quattro tip becomes a commit pin here, so the package is versioned, +# checksummed and built exactly like a release, just more often. The r-number +# is the branch's total commit count, not the distance from the last tag: the +# published history used the total, and pacman must never see it go down. +_commit=ee8ebf615dcbf9e1a126b22a894f907d6795a9a5 pkgdesc='Omarchy user defaults, /etc/skel content, fonts, plymouth theme, and support helpers (quattro branch tip)' # Arch-specific because the shipped /etc tree is not the same on every # architecture: the Limine, mkinitcpio, zram and oomd drop-ins belong to the @@ -109,30 +113,16 @@ _etc_override_paths=( 'etc/plymouth/plymouthd.conf' ) -# Source: quattro branch tip by default; set OMARCHY_SRC=/path/to/checkout -# to build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX). +# Source: the pinned quattro commit. Set OMARCHY_SRC=/path/to/checkout to +# build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX); +# the arrays are emptied below so nothing is downloaded in that case. +source=("omarchy::git+https://github.com/basecamp/omarchy.git#commit=${_commit}") +sha256sums=('4f55a96e32143c766f925ec7f721a374c32bd20e66f9a081f5dbe0772c609ecd') if [[ -n "${OMARCHY_SRC:-}" ]]; then source=() sha256sums=() -else - source=("omarchy::git+https://github.com/basecamp/omarchy.git#branch=quattro") - sha256sums=('SKIP') fi -pkgver() { - cd "$srcdir/omarchy" - - local commit_count commit_hash - if git rev-parse --verify "${_pkgver_base_tag}^{commit}" >/dev/null 2>&1; then - commit_count=$(git rev-list --count "${_pkgver_base_tag}..HEAD") - else - commit_count=$(git rev-list --count HEAD) - fi - commit_hash=$(git rev-parse --short=7 HEAD) - - printf "%s.r%s.g%s" "$_pkgver_base" "$commit_count" "$commit_hash" -} - prepare() { if [[ -n "${OMARCHY_SRC:-}" ]]; then rm -rf "$srcdir/omarchy" diff --git a/pkgbuilds/omasnap-git/.omarchy/package.json b/pkgbuilds/omasnap-git/.omarchy/package.json new file mode 100644 index 00000000..45751235 --- /dev/null +++ b/pkgbuilds/omasnap-git/.omarchy/package.json @@ -0,0 +1,17 @@ +{ + "source": "local", + "channels": ["edge"], + "auto_merge": true, + "min_release_age": "30m", + "upstream": { + "watch": { + "git_branch": "https://github.com/omacom/omasnap.git", + "branch": "main", + "tag_pattern": "v(?P[0-9]+\\.[0-9]+\\.[0-9]+)", + "version": "{version}.r{distance}.g{commit:.7}", + "variables": { + "_commit": "{commit}" + } + } + } +} diff --git a/pkgbuilds/omasnap-git/PKGBUILD b/pkgbuilds/omasnap-git/PKGBUILD new file mode 100644 index 00000000..3c9ea2d1 --- /dev/null +++ b/pkgbuilds/omasnap-git/PKGBUILD @@ -0,0 +1,72 @@ +# Maintainer: Ryan Hughes +# The main-branch build of omasnap. Pinned by the upstream watch in +# .omarchy/package.json (bin/sync-upstream): every main tip becomes a commit +# pin here, versioned .r.g so it sorts above the +# tagged release it follows and below the next one. + +pkgname=omasnap-git +pkgver=1.21.0.r15.geb22bfe +pkgrel=1 +_commit=eb22bfe5b79a2235f9bf5a8ffd026bc882969c1e +pkgdesc="Native Wayland screenshot and annotation overlay for Hyprland (main branch)" +arch=('x86_64' 'aarch64') +url="https://github.com/omacom/omasnap" +license=('MIT' 'OFL-1.1') +depends=( + 'hyprland' + 'layer-shell-qt' + 'qt6-base' + 'tesseract' + 'tesseract-data-eng' + 'wayland' + 'wl-clipboard' +) +makedepends=( + 'cmake' + 'git' + 'ninja' + 'pkgconf' + 'wayland-protocols' +) +provides=('omasnap') +conflicts=('omasnap') +options=('!debug') + +source=("omasnap::git+$url.git#commit=${_commit}") +sha256sums=('c8717bae97faa3798cce25038d42429185f832cd47b09f20c232f684c721952a') + +build() { + cmake -S omasnap -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr + cmake --build build --parallel +} + +check() { + # The smoke suite fsyncs its working documents under /tmp. On the CI + # droplets the build leaves about a gigabyte of dirty pages, and the + # flush that starts a few seconds into the suite makes those fsyncs stall + # long enough to overrun the suite's 5-second settle windows. Flush first. + local runtime_dir status started + started=$(date +%s%N); sync + echo "flushed dirty pages in $(( ($(date +%s%N) - started) / 1000000 )) ms" + runtime_dir=$(mktemp -d /dev/shm/omasnap-runtime.XXXXXX) + if QT_QPA_PLATFORM=offscreen QT_FORCE_STDERR_LOGGING=1 \ + XDG_RUNTIME_DIR="$runtime_dir" \ + ./build/omasnap-smoke "$srcdir/omasnap-smoke-output"; then + status=0 + else + status=$? + echo "omasnap-smoke exited with status $status" >&2 + fi + rm -r -- "$runtime_dir" + return "$status" +} + +package() { + cmake --install build --prefix "$pkgdir/usr" + install -Dm644 omasnap/README.md \ + "$pkgdir/usr/share/doc/omasnap/README.md" + install -Dm644 omasnap/LICENSE \ + "$pkgdir/usr/share/licenses/omasnap/LICENSE" +} diff --git a/tests/pinned-sources.sh b/tests/pinned-sources.sh new file mode 100755 index 00000000..63920895 --- /dev/null +++ b/tests/pinned-sources.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Every git source in the repository names an immutable commit or a tag. +# +# A source that follows a branch ("#branch=quattro", or no fragment at all) +# produces a package whose contents depend on when it was built, and nothing +# in this repository changes when that branch moves, so the CI publish path, +# which builds what a merge touched, never rebuilds it. Packages that need to +# follow a branch declare a git_branch upstream watch instead, and the tracker +# turns each new tip into a commit pin here (docs/upstream-sources.md). +set -euo pipefail +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +PKGBUILDS_DIR=${PKGBUILDS_DIR:-$BUILD_ROOT/pkgbuilds} + +failures=0 +checked=0 +for pkgdir in "$PKGBUILDS_DIR"/*/; do + [[ -f "$pkgdir/PKGBUILD" ]] || continue + package=$(basename "$pkgdir") + for arch in x86_64 aarch64; do + # Sourced the way the build tooling reads recipes: CARCH set, the local + # source override unset, so conditional and arch-suffixed arrays count. + sources=$(cd "$pkgdir" && env -u OMARCHY_SRC CARCH="$arch" bash -c ' + source PKGBUILD >/dev/null 2>&1 + printf "%s\n" "${source[@]}" "${source_x86_64[@]}" "${source_aarch64[@]}"' 2>/dev/null) || { + echo "FAIL: $package: PKGBUILD could not be sourced for $arch" + failures=$((failures + 1)) + continue + } + while IFS= read -r entry; do + [[ -n "$entry" ]] || continue + url="${entry#*::}" + [[ "$url" == git+* ]] || continue + checked=$((checked + 1)) + case "$url" in + *'#commit='*|*'#tag='*) ;; + *) + echo "FAIL: $package ($arch): git source is not pinned to a commit or tag: $url" + failures=$((failures + 1)) + ;; + esac + done <<<"$sources" + done +done + +if ((failures)); then + echo "$failures unpinned git source(s). Pin with #commit= (and a git_branch upstream watch to move the pin), or #tag= with a checksum." + exit 1 +fi +echo "PASS: $checked git source(s) across pkgbuilds/ are pinned to a commit or tag" diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index bb9c8dc8..4861e16b 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -178,6 +178,85 @@ b2sums=('old' 'local-b2') expected = subprocess.check_output(['git', '-c', 'core.abbrev=no', '-C', str(repo), 'archive', '--format', 'tar', 'v1.0']) self.assertEqual(archive.read_bytes(), expected) + def branch_fixture(self, fresh_tip=False): + """An upstream with two release tags and commits past the newest one, + all committed years ago; with fresh_tip, one more commit dated now.""" + repo = self.root / 'branch-upstream' + repo.mkdir() + git = ['git', '-C', str(repo), '-c', 'user.name=Test', '-c', 'user.email=test@example.test'] + old = {**os.environ, 'GIT_COMMITTER_DATE': '2020-01-01T00:00:00+00:00', 'GIT_AUTHOR_DATE': '2020-01-01T00:00:00+00:00'} + subprocess.run(['git', 'init', '-q', '-b', 'main', str(repo)], check=True) + shas = [] + def commit(index, env): + (repo / 'source').write_text(f'revision {index}') + subprocess.run([*git, 'add', '.'], check=True) + subprocess.run([*git, 'commit', '-qm', f'commit {index}'], env=env, check=True) + shas.append(subprocess.check_output([*git, 'rev-parse', 'HEAD'], text=True).strip()) + for index, tag in enumerate([None, 'v1.0.0', 'v1.1.0', None, None]): + commit(index, old) + if tag: + subprocess.run([*git, 'tag', tag], check=True) + # A newer release tagged on another branch is not something main is "past". + subprocess.run([*git, 'checkout', '-q', '-b', 'hotfix', shas[1]], check=True) + (repo / 'hotfix').write_text('x') + subprocess.run([*git, 'add', '.'], check=True) + subprocess.run([*git, 'commit', '-qm', 'hotfix'], env=old, check=True) + subprocess.run([*git, 'tag', 'v9.9.9'], check=True) + subprocess.run([*git, 'checkout', '-q', 'main'], check=True) + if fresh_tip: + commit(len(shas), os.environ) + return repo, shas + + def redirect_clone(self, repo): + command = w.subprocess.run + def redirect(args, **kwargs): + if 'clone' in args: + args = [f'file://{repo}' if arg == 'https://example.test/tool.git' else arg for arg in args] + return command(args, **kwargs) + return patch.object(w.subprocess, 'run', side_effect=redirect) + + def test_git_branch_versions_from_reachable_tag_and_shares_one_clone(self): + repo, shas = self.branch_fixture() + with self.redirect_clone(repo): + tip = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P[0-9.]+)', self.fetch.cache) + again = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache) + self.assertEqual((tip['commit'], tip['tag'], tip['version'], tip['distance'], tip['count']), (shas[-1], 'v1.1.0', '1.1.0', '2', '5')) + self.assertEqual(again['commit'], shas[-1]) + self.assertEqual(len(list(self.fetch.cache.glob('*.branch.git'))), 1, 'one clone per branch per run') + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main', 'tag_pattern': r'v(?P[0-9.]+)', + 'version': '{version}.r{distance}.g{commit:.7}', 'variables': {'_commit': '{commit}'}} + pkgver = w.candidate(watch, tip)['pkgver'] + self.assertEqual(pkgver, f'1.1.0.r2.g{shas[-1][:7]}') + self.assertEqual(w.vercmp(pkgver, '1.1.0'), 1) + self.assertEqual(w.vercmp(pkgver, '1.1.1'), -1) + with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'): + w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P[0-9.]+)', self.root / 'other-cache') + + def test_git_branch_min_age_selects_the_newest_settled_commit(self): + repo, shas = self.branch_fixture(fresh_tip=True) + with self.redirect_clone(repo): + tip = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache) + settled = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P[0-9.]+)', self.fetch.cache, min_age=3600) + nothing = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache, min_age=10 ** 9) + self.assertEqual(tip['commit'], shas[-1], 'no window: the fresh tip') + self.assertEqual((settled['commit'], settled['distance'], settled['count']), (shas[-2], '2', '5'), 'one hour window: the commit before it') + self.assertIsNone(nothing, 'a window older than every commit selects nothing') + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + with self.redirect_clone(repo): + self.assertEqual(w.discover(watch, self.fetch, min_age=10 ** 9), []) + self.assertEqual(w.discover(watch, self.fetch, min_age=3600)[0]['values']['commit'], shas[-2]) + + def test_git_branch_tag_template_requires_tag_pattern(self): + base = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + w.validate({**base, 'version': '{date}.r{count}'}) + w.validate({**base, 'tag_pattern': r'v(?P[0-9.]+)', 'version': '{version}.r{distance}.g{commit:.7}'}) + for extra in [{'version': '{version}.r{distance}'}, {'version': '{tag}'}, {'tag_pattern': 'v[0-9.]+'}, + {'tag_pattern': 7}, {'tag_pattern': ''}]: + with self.subTest(extra=extra), self.assertRaises(ValueError): + w.validate({**base, **extra}) + with self.assertRaisesRegex(ValueError, 'only applies'): + w.validate({'github': 'owner/tool', 'pattern': r'v(?P[0-9.]+)', 'tag_pattern': r'v(?P[0-9.]+)'}) + def test_invalid_optional_metadata_fails_validation(self): valid = {'github': 'owner/tool', 'pattern': r'v(?P[0-9.]+)'} for extra in [{'variables': []}, {'fields': {'commit': 3}}, {'submodules': {'pkgver': 'libs/common'}}, From 7fe542cde74581877da7d97e52fead74a00bba20 Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Tue, 22 Sep 2026 01:36:40 -0400 Subject: [PATCH 2/4] Keep branch syncs together and simplify tracker setup --- .github/workflows/sync-rebuilds.yml | 20 +----- .github/workflows/sync-upstream.yml | 23 +------ .github/workflows/track-branches.yml | 34 ++++------ README.md | 8 +-- bin/sync-upstream | 24 ++++++- docs/upstream-sources.md | 32 ++++++++-- helpers/upstream-watch.py | 27 +++----- tests/upstream-watch.py | 95 +++++++++++++++++++++++++--- 8 files changed, 165 insertions(+), 98 deletions(-) diff --git a/.github/workflows/sync-rebuilds.yml b/.github/workflows/sync-rebuilds.yml index a921e83b..4d8af048 100644 --- a/.github/workflows/sync-rebuilds.yml +++ b/.github/workflows/sync-rebuilds.yml @@ -68,27 +68,11 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - # App token rather than GITHUB_TOKEN so the PR's build and test runs - # start without a maintainer approving them (see sync-upstream.yml). - - name: Mint the bot token - if: steps.changes.outputs.has_changes == 'true' - id: app - env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - # Without the App configured this falls back to GITHUB_TOKEN below, - # which still opens the PR; a maintainer then has to approve its - # workflow runs by hand, as before. - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.PKGS_BOT_APP_ID }} - private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} - continue-on-error: true - - name: Create Pull Request if: steps.changes.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + uses: peter-evans/create-pull-request@v7 with: - token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} commit-message: 'chore: rebuild against updated dependencies' title: 'chore: rebuild against updated dependencies' body: | diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 971d461b..5eaa5887 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -72,30 +72,11 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - # A PR opened with GITHUB_TOKEN gets its build and test runs held - # until a maintainer clicks "Approve workflows to run"; one opened by - # the App builds on its own, so the reviewer sees a green (or red) PR - # instead of a pending one. The App only opens the PR: merging stays - # a human decision in this lane. - - name: Mint the bot token - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: app - env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - # Without the App configured this falls back to GITHUB_TOKEN below, - # which still opens the PR; a maintainer then has to approve its - # workflow runs by hand, as before. - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.PKGS_BOT_APP_ID }} - private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} - continue-on-error: true - - name: Create Pull Request if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + uses: peter-evans/create-pull-request@v7 with: - token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} commit-message: 'chore: sync upstream releases' title: 'chore: sync upstream releases' body: | diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml index 5413855a..cf063b24 100644 --- a/.github/workflows/track-branches.yml +++ b/.github/workflows/track-branches.yml @@ -39,6 +39,16 @@ jobs: contents: read steps: + - name: Require the bot App + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + PKGS_BOT_PRIVATE_KEY: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + run: | + if [[ -z "$PKGS_BOT_APP_ID" || -z "$PKGS_BOT_PRIVATE_KEY" ]]; then + echo "::error::Set PKGS_BOT_APP_ID and PKGS_BOT_PRIVATE_KEY for a GitHub App installed on this repository with Contents: write and Pull requests: write." + exit 1 + fi + - name: Checkout repository uses: actions/checkout@v4 with: @@ -93,19 +103,6 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi - # No fallback to GITHUB_TOKEN here: a PR it opened would sit with its - # checks held, and an auto-merge it enabled would land without running - # publish.yml. Better to fail loudly than to pin quietly. - - name: Require the bot App - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - run: | - if [[ -z "$PKGS_BOT_APP_ID" ]]; then - echo "::error::PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY are not set. Create a GitHub App with Contents: write and Pull requests: write, install it on this repository, and store its id and private key as those secrets." - exit 1 - fi - - name: Mint the bot token if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} id: app @@ -126,7 +123,7 @@ jobs: echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" - name: Open or update the tracking PR - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' && steps.app.outcome == 'success' }} id: pr uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: @@ -136,8 +133,8 @@ jobs: body: | Automated pin of packages that follow a moving upstream branch (`"auto_merge": true` in `.omarchy/package.json`). Each package's - `_commit` now points at the branch tip that has been there for at - least its `min_release_age`. + `_commit` now points at the branch tip. Fresh tips wait until + their commit timestamp is at least `min_release_age` old. This PR auto-merges once the build checks pass. A failing build leaves it open; the next tracker run replaces it with the newer tip. @@ -159,10 +156,7 @@ jobs: echo "auto-merge already enabled on #$PR" exit 0 fi - # A PR whose checks all reused existing artifacts can be clean - # before this step runs; GitHub then refuses --auto, so merge it. - gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" \ - || gh pr merge --merge "$PR" -R "${{ github.repository }}" + gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" - name: Notify Basecamp on failure if: failure() && env.BASECAMP_CHATBOT_URL != '' diff --git a/README.md b/README.md index 5e216f7f..97421130 100644 --- a/README.md +++ b/README.md @@ -891,15 +891,15 @@ The repository includes GitHub workflows and systemd services for automated rele 1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes on the reviewed lane. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red. 2. **sync-rebuilds.yml** (Every 6 hours): Bumps pkgrel for packages whose `rebuild_on` dependencies have moved in the official repositories and opens a PR. -3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the newest tip of its watched branch that has sat there for `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. +3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the tip of its watched branch once its commit timestamp clears `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all, including targeted syncs. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. -The sync PRs are opened with a GitHub App token (`PKGS_BOT_APP_ID` and +The tracking PR is opened with a GitHub App token (`PKGS_BOT_APP_ID` and `PKGS_BOT_PRIVATE_KEY` secrets; the App needs Contents and Pull requests write on this repository). A PR opened with the workflow's own `GITHUB_TOKEN` has its build and test runs held until a maintainer approves them, and an auto-merge it enabled would land without running the publish workflow. The -reviewed workflows fall back to `GITHUB_TOKEN` when the App is not configured -(and then need that click); the tracker refuses to run without it. +tracker requires both App secrets before it runs. The reviewed sync workflows +continue to use `GITHUB_TOKEN` and require maintainer approval as before. To approve builds for an unvouched contributor's PR, apply **`build-approved`**. Until approval, the PR shows **Awaiting build approval** and its required diff --git a/bin/sync-upstream b/bin/sync-upstream index a08f184d..5aae8309 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -1093,11 +1093,12 @@ enforce_branch_lockstep() { groups["$key"]+="$package " done for key in "${!groups[@]}"; do - local members commits + local members commits pin read -r -a members <<<"${groups[$key]}" (( ${#members[@]} > 1 )) || continue commits=$(for package in "${members[@]}"; do - grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" + pin=$(grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" || true) + printf '%s\n' "${pin:-missing:$package}" done | sort -u | grep -c .) (( commits > 1 )) || continue print_error "Lockstep violation on $key: ${members[*]} pin different commits; restoring all of them" @@ -1115,17 +1116,34 @@ enforce_branch_lockstep() { sync_in_lane() { local package="$1" package_dir="$PKGBUILDS_DIR/$1" + snapshot_package "$package" if [[ -d "$package_dir" ]] && ! package_in_lane "$package_dir" "$LANE"; then print_info "Skipping $package: not in the $LANE lane" ((++SKIPPED)) return 0 fi - snapshot_package "$package" sync_package "$package" } if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then SPECIFIC_MODE=true + # A targeted run is still a branch update: include every sibling watching + # the same branch, otherwise the lockstep check never sees the omitted one. + declare -A selected=() selected_branches=() + for package in "${SPECIFIC_PACKAGES[@]}"; do + selected["$package"]=1 + key=$(branch_watch_key "$PKGBUILDS_DIR/$package" || true) + [[ -z "$key" ]] || selected_branches["$key"]=1 + done + for package_dir in "$PKGBUILDS_DIR"/*; do + [[ -f "$package_dir/PKGBUILD" ]] || continue + package=${package_dir##*/} + [[ -z "${selected[$package]:-}" ]] || continue + key=$(branch_watch_key "$package_dir" || true) + if [[ -n "$key" && -n "${selected_branches[$key]:-}" ]]; then + SPECIFIC_PACKAGES+=("$package") + fi + done for package in "${SPECIFIC_PACKAGES[@]}"; do sync_in_lane "$package" done diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index 79cff76b..1749ee9d 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -72,16 +72,40 @@ uses it. The Omarchy dev pair uses `{version}.r{count}.g{commit:.7}` instead because its published history counted every commit and the number must never go down. -`min_release_age` on a branch watch selects the newest commit that has been on -the branch for at least that long, so a burst of pushes builds once after it -settles rather than once per push. `BYPASS_MIN_RELEASE_AGE=1` takes the tip. +`min_release_age` holds a branch tip until its commit timestamp is old enough. +A fresh tip leaves the existing pin alone; the watch never walks backward to +an older commit. This uses Git's committer date, not the time a commit was +pushed. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold. Packages marked `"auto_merge": true` ride the unattended lane (`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened and auto-merged as soon as the build checks pass. `bin/sync-upstream --lane reviewed|auto-merge|all` selects a lane; the scheduled workflows each pass their own. Packages that pin the same branch move in lockstep: if one of them fails -to update, the run restores the others and reports the group as failed. +to update, the run restores the others and reports the group as failed. A +targeted sync includes the other packages watching that branch, so requesting +only `omarchy-dev` also updates `omarchy-settings-dev`. + +### Enable unattended branch updates + +The schedule already runs in GitHub Actions; no server cron job is needed. +It needs a GitHub App identity so its PRs trigger builds and its merges trigger +publishing without manual approval: + +1. [Create an organization GitHub App](https://github.com/organizations/omacom/settings/apps/new). + Use this repository's URL as the homepage, disable webhooks, and grant only + repository **Contents: Read and write** and **Pull requests: Read and write** + (Metadata read access is automatic). Limit installation to this organization. +2. Install the App on **omacom/omarchy-pkgs** only. +3. Generate a private key from the App's settings. In the repository's + [Actions secrets](https://github.com/omacom/omarchy-pkgs/settings/secrets/actions), + save the App ID as `PKGS_BOT_APP_ID` and the PEM key contents as + `PKGS_BOT_PRIVATE_KEY`. +4. Keep **Allow auto-merge** enabled and require `result`, `self-tests`, and + `build-isolation` on `master`; the App does not need a protection bypass. +5. After merging the tracker, run **Track upstream branches** once from Actions + to verify that its PR builds, auto-merges, and starts **Publish merged packages**. + Subsequent runs happen every two hours. Checksums retain their algorithms (SHA256, SHA512, BLAKE2, etc.) and source order. Changed git sources are hashed with makepkg's git-archive convention. Unchanged diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index 05828f4b..f8468abd 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -184,10 +184,9 @@ def matches(watch, text, extra=None, full=False): yield candidate(watch, {**(extra or {}), **match.groupdict()}) -def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): - """Describe the newest commit on an upstream branch that has sat there for - at least min_age seconds (the branch analogue of "the newest release older - than the window ships"): commit, total count, date, and with a tag_pattern +def git_branch_tip(url, branch, tag_pattern, cache): + """Describe the current tip of an upstream branch: + commit, total count, date, and with a tag_pattern the newest release tag reachable from it plus the distance from that tag, so a branch build can be versioned .r.g, above the release it follows and below the next one, the way a pkgver() function would. @@ -195,7 +194,8 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): One blobless single-branch clone per (url, branch) per run, shared by every package that tracks it, so two recipes pinned from one clone always see the same commit. The clone is read with git only; nothing in it runs. - Returns None when every commit is younger than the window. + select_release applies the age hold to this tip, without walking back + into history (which could select a commit from a merged side branch). """ https(url) key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest() @@ -205,14 +205,7 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True) scratch.replace(work) git = ["git", "-C", str(work)] - selector = ["HEAD"] - if min_age: - cutoff = (now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(seconds=min_age) - selector = ["-1", f"--before={cutoff.isoformat()}", "HEAD"] - commit = run([*git, "rev-list", *selector], text=True).split()[:1] - if not commit: - return None - commit = commit[0] + commit = run([*git, "rev-parse", "HEAD"], text=True).strip() if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("branch tip is not a commit") count = run([*git, "rev-list", "--count", commit], text=True).strip() @@ -238,7 +231,7 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): return values -def discover(watch, fetch, min_age=0): +def discover(watch, fetch): provider = validate(watch) feed = watch[provider] results = [] @@ -265,9 +258,7 @@ def discover(watch, fetch, min_age=0): for tag, commit in tags.items(): results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True)) elif provider == "git_branch": - tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age) - if tip is None: - return [] # nothing has settled for min_age yet: wait, not an error + tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache) results.append(candidate(watch, tip)) elif provider == "npm": data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe="")) @@ -543,7 +534,7 @@ def sync(package, fetch, min_age=0, check=False): original = path.read_text() before = read_recipe(path) bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1" - release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass) + release = select_release(discover(watch, fetch), min_age, bypass=bypass) if release is None: return {"status": "skipped", "reason": "minimum release age"} current = scalar(before, "pkgver") diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index 4861e16b..2921f7ce 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -232,19 +232,94 @@ b2sums=('old' 'local-b2') with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'): w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P[0-9.]+)', self.root / 'other-cache') - def test_git_branch_min_age_selects_the_newest_settled_commit(self): + def test_git_branch_min_age_holds_the_tip_instead_of_selecting_history(self): repo, shas = self.branch_fixture(fresh_tip=True) - with self.redirect_clone(repo): - tip = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache) - settled = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P[0-9.]+)', self.fetch.cache, min_age=3600) - nothing = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache, min_age=10 ** 9) - self.assertEqual(tip['commit'], shas[-1], 'no window: the fresh tip') - self.assertEqual((settled['commit'], settled['distance'], settled['count']), (shas[-2], '2', '5'), 'one hour window: the commit before it') - self.assertIsNone(nothing, 'a window older than every commit selects nothing') watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} with self.redirect_clone(repo): - self.assertEqual(w.discover(watch, self.fetch, min_age=10 ** 9), []) - self.assertEqual(w.discover(watch, self.fetch, min_age=3600)[0]['values']['commit'], shas[-2]) + releases = w.discover(watch, self.fetch) + self.assertEqual(w.select_release(releases)['values']['commit'], shas[-1]) + self.assertIsNone(w.select_release(releases, min_age=3600)) + self.assertEqual(w.select_release(releases, min_age=3600, bypass=True)['values']['commit'], shas[-1]) + + def branch_sync_fixture(self): + repo, shas = self.branch_fixture() + for directory in ['bin', 'helpers']: + shutil.copytree(ROOT / directory, self.root / directory) + for name in ['dev', 'settings-dev']: + package = self.root / 'pkgbuilds' / name + (package / '.omarchy').mkdir(parents=True) + (package / '.omarchy/package.json').write_text(json.dumps({ + 'source': 'local', 'auto_merge': True, 'upstream': {'watch': { + 'git_branch': 'https://example.test/tool.git', 'branch': 'main', + 'tag_pattern': r'v(?P[0-9.]+)', + 'version': '{version}.r{count}.g{commit:.7}', + 'variables': {'_commit': '{commit}'}}}})) + (package / 'PKGBUILD').write_text(f'''pkgname={name} +pkgver=1.0.0 +pkgrel=1 +_commit={shas[1]} +arch=('any') +source=("tool::git+https://example.test/tool.git#commit=${{_commit}}") +sha256sums=('old') +''') + stub = self.root / 'stub' + stub.mkdir() + git = stub / 'git' + git.write_text('''#!/usr/bin/env python3 +import os, sys +args = [os.environ['BRANCH_FIXTURE'] if arg == 'https://example.test/tool.git' else arg for arg in sys.argv[1:]] +os.execv(os.environ['REAL_GIT'], ['git', *args]) +''') + git.chmod(0o755) + env = {**os.environ, 'PATH': str(stub) + os.pathsep + os.environ['PATH'], + 'BRANCH_FIXTURE': f'file://{repo}', 'REAL_GIT': shutil.which('git')} + def sync(*args): + return subprocess.run([str(self.root / 'bin/sync-upstream'), *args], + env=env, text=True, capture_output=True) + return self.root / 'pkgbuilds', shas[-1], sync + + def test_targeted_branch_sync_updates_siblings_and_then_noops(self): + packages, tip, sync = self.branch_sync_fixture() + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + for package in packages.iterdir(): + recipe = w.read_recipe(package / 'PKGBUILD') + self.assertEqual(w.scalar(recipe, '_commit'), tip) + self.assertEqual(w.scalar(recipe, 'pkgver'), f'1.1.0.r5.g{tip[:7]}') + self.assertRegex(recipe['sha256sums'][0], r'^[0-9a-f]{64}$') + self.assertIn('Updated: 2', result.stdout) + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('Updated: 0', result.stdout) + + def test_branch_sync_rolls_back_when_a_sibling_fails(self): + packages, tip, sync = self.branch_sync_fixture() + broken = packages / 'settings-dev/PKGBUILD' + broken.write_text(broken.read_text().replace("sha256sums=('old')", 'sha256sums=()')) + before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')} + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn('Lockstep violation', result.stdout + result.stderr) + self.assertIn('Updated: 0', result.stdout) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + + def test_reviewed_lane_leaves_auto_merge_packages_untouched(self): + packages, tip, sync = self.branch_sync_fixture() + before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')} + result = sync('--lane', 'reviewed') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('Updated: 0', result.stdout) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + + def test_different_lanes_cannot_split_a_branch_pair(self): + packages, tip, sync = self.branch_sync_fixture() + metadata = packages / 'settings-dev/.omarchy/package.json' + metadata.write_text(metadata.read_text().replace('"auto_merge": true', '"auto_merge": false')) + before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')} + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn('Lockstep violation', result.stdout + result.stderr) + self.assertEqual(before, {p: p.read_bytes() for p in before}) def test_git_branch_tag_template_requires_tag_pattern(self): base = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} From 13ed2e9f8dfb304df0d1383c3de9e5f3241b38b8 Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Tue, 22 Sep 2026 15:35:53 -0400 Subject: [PATCH 3/4] Use the existing controller PAT for branch tracking --- .github/workflows/track-branches.yml | 32 ++++++++++------------------ README.md | 14 ++++++------ docs/upstream-sources.md | 25 +++++++++++----------- 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml index cf063b24..e6612204 100644 --- a/.github/workflows/track-branches.yml +++ b/.github/workflows/track-branches.yml @@ -9,11 +9,10 @@ name: Track upstream branches # and the merge publishes the artifacts. A tip that fails to build stays an # unmerged red PR that the next tick supersedes. # -# The PR is opened with a GitHub App token, not GITHUB_TOKEN: a pull request -# created with the workflow token gets its CI runs held for manual approval, -# and an auto-merge it enabled would not fire the publish workflow. The App -# needs Contents: write and Pull requests: write on this repository; its id -# and private key live in the PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY secrets. +# The PR and auto-merge use the PAT in PKGS_BOT_TOKEN so they trigger the +# build and publish workflows. The built-in GITHUB_TOKEN cannot drive this +# unattended chain. The PAT needs Contents: write and Pull requests: write +# on this repository, and its owner must be trusted by the build workflow. on: schedule: @@ -39,13 +38,12 @@ jobs: contents: read steps: - - name: Require the bot App + - name: Require the tracking token env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - PKGS_BOT_PRIVATE_KEY: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + PKGS_BOT_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} run: | - if [[ -z "$PKGS_BOT_APP_ID" || -z "$PKGS_BOT_PRIVATE_KEY" ]]; then - echo "::error::Set PKGS_BOT_APP_ID and PKGS_BOT_PRIVATE_KEY for a GitHub App installed on this repository with Contents: write and Pull requests: write." + if [[ -z "$PKGS_BOT_TOKEN" ]]; then + echo "::error::Set PKGS_BOT_TOKEN to a PAT with Contents: write and Pull requests: write on this repository, owned by an account trusted to trigger builds." exit 1 fi @@ -103,14 +101,6 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi - - name: Mint the bot token - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: app - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.PKGS_BOT_APP_ID }} - private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} - # The PR title names what moved, so the merged history reads like a # changelog: "Track upstream branches: omarchy-dev 4.0.0.r6520.g1a2b3c4, ...". - name: Describe the pins @@ -123,11 +113,11 @@ jobs: echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" - name: Open or update the tracking PR - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' && steps.app.outcome == 'success' }} + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} id: pr uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: - token: ${{ steps.app.outputs.token }} + token: ${{ secrets.PKGS_BOT_TOKEN }} commit-message: ${{ steps.describe.outputs.title }} title: ${{ steps.describe.outputs.title }} body: | @@ -148,7 +138,7 @@ jobs: - name: Enable auto-merge if: ${{ !cancelled() && steps.pr.outputs.pull-request-number != '' }} env: - GH_TOKEN: ${{ steps.app.outputs.token }} + GH_TOKEN: ${{ secrets.PKGS_BOT_TOKEN }} PR: ${{ steps.pr.outputs.pull-request-number }} run: | # Idempotent across re-runs of an updated PR: enabling twice errors. diff --git a/README.md b/README.md index 97421130..4f1cba0d 100644 --- a/README.md +++ b/README.md @@ -893,13 +893,13 @@ The repository includes GitHub workflows and systemd services for automated rele 2. **sync-rebuilds.yml** (Every 6 hours): Bumps pkgrel for packages whose `rebuild_on` dependencies have moved in the official repositories and opens a PR. 3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the tip of its watched branch once its commit timestamp clears `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all, including targeted syncs. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. -The tracking PR is opened with a GitHub App token (`PKGS_BOT_APP_ID` and -`PKGS_BOT_PRIVATE_KEY` secrets; the App needs Contents and Pull requests -write on this repository). A PR opened with the workflow's own `GITHUB_TOKEN` -has its build and test runs held until a maintainer approves them, and an -auto-merge it enabled would land without running the publish workflow. The -tracker requires both App secrets before it runs. The reviewed sync workflows -continue to use `GITHUB_TOKEN` and require maintainer approval as before. +The tracking PR and auto-merge use the PAT stored in `PKGS_BOT_TOKEN`, with +Contents and Pull requests write access to this repository and an owner trusted +to trigger builds. The existing controller PAT can be reused. No GitHub App is +required. The built-in Actions `GITHUB_TOKEN` cannot drive the unattended +build-and-publish chain, so the tracker requires this secret before it runs. +The reviewed sync workflows continue to use `GITHUB_TOKEN` and require +maintainer approval as before. See [setup instructions](docs/upstream-sources.md#enable-unattended-branch-updates). To approve builds for an unvouched contributor's PR, apply **`build-approved`**. Until approval, the PR shows **Awaiting build approval** and its required diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index 1749ee9d..43973366 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -89,21 +89,20 @@ only `omarchy-dev` also updates `omarchy-settings-dev`. ### Enable unattended branch updates The schedule already runs in GitHub Actions; no server cron job is needed. -It needs a GitHub App identity so its PRs trigger builds and its merges trigger -publishing without manual approval: +It uses a personal access token so its PRs trigger builds and its merges trigger +publishing without manual approval. No GitHub App is required. -1. [Create an organization GitHub App](https://github.com/organizations/omacom/settings/apps/new). - Use this repository's URL as the homepage, disable webhooks, and grant only - repository **Contents: Read and write** and **Pull requests: Read and write** - (Metadata read access is automatic). Limit installation to this organization. -2. Install the App on **omacom/omarchy-pkgs** only. -3. Generate a private key from the App's settings. In the repository's +1. Use a fine-grained PAT with access to **omacom/omarchy-pkgs** and repository + **Contents: Read and write** and **Pull requests: Read and write** permissions. + Its owner must be trusted by the build workflow (for example, a collaborator). + The existing controller PAT can be reused when it has these permissions. +2. In the repository's [Actions secrets](https://github.com/omacom/omarchy-pkgs/settings/secrets/actions), - save the App ID as `PKGS_BOT_APP_ID` and the PEM key contents as - `PKGS_BOT_PRIVATE_KEY`. -4. Keep **Allow auto-merge** enabled and require `result`, `self-tests`, and - `build-isolation` on `master`; the App does not need a protection bypass. -5. After merging the tracker, run **Track upstream branches** once from Actions + save the PAT as `PKGS_BOT_TOKEN`. Update this secret when the token is rotated + or expires. The built-in Actions `GITHUB_TOKEN` cannot run this unattended chain. +3. Keep **Allow auto-merge** enabled and require `result`, `self-tests`, and + `build-isolation` on `master`; the tracker does not request a protection bypass. +4. After merging the tracker, run **Track upstream branches** once from Actions to verify that its PR builds, auto-merges, and starts **Publish merged packages**. Subsequent runs happen every two hours. From 3de5f0af4bf7857f547f622ac31ca225612e9133 Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Sun, 27 Sep 2026 12:41:43 -0400 Subject: [PATCH 4/4] Pin the tracked packages to today's upstream tips The first pins (quattro ee8ebf6, omasnap eb22bfe) date from 2026-09-21. Edge has since published omarchy-dev from quattro 7b336b1 through a manual rebuild, so merging the old pin would ship older code under a higher version. Moved with the tracker's own code: bin/sync-upstream --lane auto-merge (BYPASS_MIN_RELEASE_AGE=1) --- pkgbuilds/omarchy-dev/PKGBUILD | 6 +++--- pkgbuilds/omarchy-settings-dev/PKGBUILD | 6 +++--- pkgbuilds/omasnap-git/PKGBUILD | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkgbuilds/omarchy-dev/PKGBUILD b/pkgbuilds/omarchy-dev/PKGBUILD index c2e70eac..eec9e2b0 100644 --- a/pkgbuilds/omarchy-dev/PKGBUILD +++ b/pkgbuilds/omarchy-dev/PKGBUILD @@ -1,13 +1,13 @@ # Maintainer: Ryan Hughes pkgname='omarchy-dev' -pkgver=4.0.0.r6514.gee8ebf6 +pkgver=4.0.0.r6646.gbf44355 pkgrel=1 # Pinned by the upstream watch in .omarchy/package.json (bin/sync-upstream): # every quattro tip becomes a commit pin here, so the package is versioned, # checksummed and built exactly like a release, just more often. The r-number # is the branch's total commit count, not the distance from the last tag: the # published history used the total, and pacman must never see it go down. -_commit=ee8ebf615dcbf9e1a126b22a894f907d6795a9a5 +_commit=bf44355a985b1bb82cb33df47fc17f765f3b2c4f pkgdesc='Beautiful, modern, and opinionated Arch Linux by DHH (quattro branch tip)' # The payload is architecture-independent, but the dependency set is not: the # boot stack differs per architecture (see depends_x86_64 / depends_aarch64), @@ -81,7 +81,7 @@ makedepends=( # build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX); # the arrays are emptied below so nothing is downloaded in that case. source=("omarchy::git+https://github.com/basecamp/omarchy.git#commit=${_commit}") -sha256sums=('4f55a96e32143c766f925ec7f721a374c32bd20e66f9a081f5dbe0772c609ecd') +sha256sums=('7a90239fd67124279817d614fa6aa19554f40796c523ba698669fbf57fa41668') if [[ -n "${OMARCHY_SRC:-}" ]]; then source=() sha256sums=() diff --git a/pkgbuilds/omarchy-settings-dev/PKGBUILD b/pkgbuilds/omarchy-settings-dev/PKGBUILD index 2fb10166..85968ac8 100644 --- a/pkgbuilds/omarchy-settings-dev/PKGBUILD +++ b/pkgbuilds/omarchy-settings-dev/PKGBUILD @@ -1,13 +1,13 @@ # Maintainer: Ryan Hughes pkgname='omarchy-settings-dev' -pkgver=4.0.0.r6514.gee8ebf6 +pkgver=4.0.0.r6646.gbf44355 pkgrel=1 # Pinned by the upstream watch in .omarchy/package.json (bin/sync-upstream): # every quattro tip becomes a commit pin here, so the package is versioned, # checksummed and built exactly like a release, just more often. The r-number # is the branch's total commit count, not the distance from the last tag: the # published history used the total, and pacman must never see it go down. -_commit=ee8ebf615dcbf9e1a126b22a894f907d6795a9a5 +_commit=bf44355a985b1bb82cb33df47fc17f765f3b2c4f pkgdesc='Omarchy user defaults, /etc/skel content, fonts, plymouth theme, and support helpers (quattro branch tip)' # Arch-specific because the shipped /etc tree is not the same on every # architecture: the Limine, mkinitcpio, zram and oomd drop-ins belong to the @@ -117,7 +117,7 @@ _etc_override_paths=( # build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX); # the arrays are emptied below so nothing is downloaded in that case. source=("omarchy::git+https://github.com/basecamp/omarchy.git#commit=${_commit}") -sha256sums=('4f55a96e32143c766f925ec7f721a374c32bd20e66f9a081f5dbe0772c609ecd') +sha256sums=('7a90239fd67124279817d614fa6aa19554f40796c523ba698669fbf57fa41668') if [[ -n "${OMARCHY_SRC:-}" ]]; then source=() sha256sums=() diff --git a/pkgbuilds/omasnap-git/PKGBUILD b/pkgbuilds/omasnap-git/PKGBUILD index 3c9ea2d1..93b673a5 100644 --- a/pkgbuilds/omasnap-git/PKGBUILD +++ b/pkgbuilds/omasnap-git/PKGBUILD @@ -5,9 +5,9 @@ # tagged release it follows and below the next one. pkgname=omasnap-git -pkgver=1.21.0.r15.geb22bfe +pkgver=1.21.0.r76.g614cdf5 pkgrel=1 -_commit=eb22bfe5b79a2235f9bf5a8ffd026bc882969c1e +_commit=614cdf55b42a43c1dcee2c85cd01e4764a52bdae pkgdesc="Native Wayland screenshot and annotation overlay for Hyprland (main branch)" arch=('x86_64' 'aarch64') url="https://github.com/omacom/omasnap" @@ -33,7 +33,7 @@ conflicts=('omasnap') options=('!debug') source=("omasnap::git+$url.git#commit=${_commit}") -sha256sums=('c8717bae97faa3798cce25038d42429185f832cd47b09f20c232f684c721952a') +sha256sums=('952539c68a81ac373c0f2d5fc084106d3419ee3e5ec2d544d02b510a3f416484') build() { cmake -S omasnap -B build -G Ninja \