From d87686ca4f86fea3bf8717a42cc1e4a8462ec1f2 Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Mon, 21 Sep 2026 03:05:03 -0400 Subject: [PATCH] Track upstream branches as pinned releases on an unattended lane Since publishing moved to CI on merge, a package whose PKGBUILD never changes while its source moves was never rebuilt: omarchy-dev and omarchy-settings-dev followed quattro through "#branch=" and a pkgver() function, and nothing in this repository changed when quattro did. The host timers that used to notice are off, so edge fell days behind. The rule now: no git source without a commit or tag pin (tests/pinned-sources.sh, run in CI). A package that has to follow a branch declares a git_branch upstream watch, and the pin moves through the same PR/build/publish path as every other version bump. Watch (helpers/upstream-watch.py) git_branch gains tag_pattern: the newest release tag in the pinned commit's own history, exposed as {tag}/{version}/{distance}, so a branch build is versioned .r.g, above the release it follows and below the next one. One blobless clone per branch per run, shared by every package on it. min_release_age selects the newest commit older than the window, so a push burst builds once. Lane (helpers/package-metadata.sh, bin/sync-upstream --lane) "auto_merge": true moves a package from the reviewed 6-hourly sync PR to the unattended lane. Packages pinned from the same branch move together: a failure on one restores the others and fails the group, so the dev pair can never ship from two quattro commits. Tracker (.github/workflows/track-branches.yml) Every two hours: pin, open one PR with a GitHub App token, enable auto-merge. Branch protection still gates the merge on result, self-tests and build-isolation. A tip that fails to build stays an open red PR until the next tick supersedes it. The App is required: a PR opened with GITHUB_TOKEN has its checks held for approval and its auto-merge would not fire publish.yml. The reviewed workflows (sync-upstream, sync-rebuilds) open their PRs with the same App so their builds start without a maintainer clicking "Approve workflows to run"; without the App they fall back to GITHUB_TOKEN and behave as before. Recipes The dev pair pins _commit and a real sha256sum, keeps the OMARCHY_SRC override, and drops pkgver(). Its r-number stays the branch's total commit count because the published history used it and pacman must never see the version go down. omasnap-git is new: omacom/omasnap main, versioned .r.g, provides/conflicts omasnap. --- .github/workflows/sync-rebuilds.yml | 20 +- .github/workflows/sync-upstream.yml | 29 ++- .github/workflows/test.yml | 1 + .github/workflows/track-branches.yml | 177 ++++++++++++++++++ README.md | 20 +- bin/sync-upstream | 87 ++++++++- docs/upstream-sources.md | 38 +++- helpers/package-metadata.sh | 35 ++++ helpers/upstream-watch.py | 84 ++++++++- pkgbuilds/omarchy-dev/.omarchy/package.json | 18 +- pkgbuilds/omarchy-dev/PKGBUILD | 34 ++-- .../.omarchy/package.json | 18 +- pkgbuilds/omarchy-settings-dev/PKGBUILD | 36 ++-- pkgbuilds/omasnap-git/.omarchy/package.json | 17 ++ pkgbuilds/omasnap-git/PKGBUILD | 72 +++++++ tests/pinned-sources.sh | 49 +++++ tests/upstream-watch.py | 79 ++++++++ 17 files changed, 746 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/track-branches.yml create mode 100644 pkgbuilds/omasnap-git/.omarchy/package.json create mode 100644 pkgbuilds/omasnap-git/PKGBUILD create mode 100755 tests/pinned-sources.sh diff --git a/.github/workflows/sync-rebuilds.yml b/.github/workflows/sync-rebuilds.yml index 4d8af04..a921e83 100644 --- a/.github/workflows/sync-rebuilds.yml +++ b/.github/workflows/sync-rebuilds.yml @@ -68,11 +68,27 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi + # App token rather than GITHUB_TOKEN so the PR's build and test runs + # start without a maintainer approving them (see sync-upstream.yml). + - name: Mint the bot token + if: steps.changes.outputs.has_changes == 'true' + id: app + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + # Without the App configured this falls back to GITHUB_TOKEN below, + # which still opens the PR; a maintainer then has to approve its + # workflow runs by hand, as before. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.PKGS_BOT_APP_ID }} + private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + continue-on-error: true + - name: Create Pull Request if: steps.changes.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} commit-message: 'chore: rebuild against updated dependencies' title: 'chore: rebuild against updated dependencies' body: | diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 2e69136..971d461 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -47,11 +47,13 @@ jobs: useradd -m -u "$HOST_UID" -g "$HOST_GID" runner chown -R runner:runner /workspace/pkgbuilds + # The reviewed lane only: packages marked auto_merge ride + # track-branches.yml, which merges without a human. if [[ -n "${PACKAGES:-}" ]]; then read -r -a package_args <<< "$PACKAGES" - runuser -u runner -- ./bin/sync-upstream "${package_args[@]}" + runuser -u runner -- ./bin/sync-upstream --lane reviewed "${package_args[@]}" else - runuser -u runner -- ./bin/sync-upstream + runuser -u runner -- ./bin/sync-upstream --lane reviewed fi ' env: @@ -70,11 +72,30 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi + # A PR opened with GITHUB_TOKEN gets its build and test runs held + # until a maintainer clicks "Approve workflows to run"; one opened by + # the App builds on its own, so the reviewer sees a green (or red) PR + # instead of a pending one. The App only opens the PR: merging stays + # a human decision in this lane. + - name: Mint the bot token + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: app + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + # Without the App configured this falls back to GITHUB_TOKEN below, + # which still opens the PR; a maintainer then has to approve its + # workflow runs by hand, as before. + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.PKGS_BOT_APP_ID }} + private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + continue-on-error: true + - name: Create Pull Request if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} commit-message: 'chore: sync upstream releases' title: 'chore: sync upstream releases' body: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bba4b01..1c30068 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,6 +58,7 @@ jobs: python tests/neovim-clipboard-tmux.py ./tests/partial-release.sh ./tests/published-build-plan.sh + ./tests/pinned-sources.sh ./tests/controller.sh ./tests/artifact-helpers.sh ./tests/limine-mkinitcpio-hook.sh diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml new file mode 100644 index 0000000..5413855 --- /dev/null +++ b/.github/workflows/track-branches.yml @@ -0,0 +1,177 @@ +name: Track upstream branches + +# The unattended lane. Packages marked "auto_merge": true follow a moving +# upstream branch (omarchy-dev and omarchy-settings-dev on quattro, omasnap-git +# on main) rather than tagged releases, so nothing in this repository changes +# when their source does. This workflow makes each new branch tip a commit pin +# in the recipe, which publish.yml then treats like any other version bump: +# the PR builds on the droplets, auto-merge lands it when `result` is green, +# and the merge publishes the artifacts. A tip that fails to build stays an +# unmerged red PR that the next tick supersedes. +# +# The PR is opened with a GitHub App token, not GITHUB_TOKEN: a pull request +# created with the workflow token gets its CI runs held for manual approval, +# and an auto-merge it enabled would not fire the publish workflow. The App +# needs Contents: write and Pull requests: write on this repository; its id +# and private key live in the PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY secrets. + +on: + schedule: + # Every 2 hours, off the hour to dodge the scheduling backlog at :00 + - cron: '35 */2 * * *' + workflow_dispatch: + inputs: + packages: + description: 'Specific packages to track (space-separated, leave empty for every auto_merge package)' + required: false + default: '' + +# One tracker at a time: two runs racing on auto/track-branches would each +# force-push their own pin over the other's. +concurrency: + group: track-branches + cancel-in-progress: false + +jobs: + track: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + # Same container as the reviewed sync: vercmp decides whether a pin is + # an upgrade with the comparator pacman uses on users' machines. + - name: Pin tracked branches to their current tips + id: sync + run: | + docker run --rm \ + -e PACKAGES="$PACKAGES" \ + -e UPSTREAM_GITHUB_TOKEN="$UPSTREAM_GITHUB_TOKEN" \ + -e HOST_UID="$(id -u)" \ + -e HOST_GID="$(id -g)" \ + -v "$PWD/bin:/workspace/bin:ro" \ + -v "$PWD/helpers:/workspace/helpers:ro" \ + -v "$PWD/pkgbuilds:/workspace/pkgbuilds" \ + -w /workspace \ + archlinux:base-devel bash -lc ' + set -euo pipefail + + pacman -Syu --noconfirm git jq python libarchive + + groupadd -g "$HOST_GID" runner + useradd -m -u "$HOST_UID" -g "$HOST_GID" runner + chown -R runner:runner /workspace/pkgbuilds + + if [[ -n "${PACKAGES:-}" ]]; then + read -r -a package_args <<< "$PACKAGES" + runuser -u runner -- ./bin/sync-upstream --lane auto-merge "${package_args[@]}" + else + runuser -u runner -- ./bin/sync-upstream --lane auto-merge + fi + ' + env: + PACKAGES: ${{ github.event.inputs.packages }} + UPSTREAM_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for changes + if: ${{ !cancelled() && steps.sync.outcome != 'skipped' }} + id: changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "has_changes=true" >> "$GITHUB_OUTPUT" + git status --porcelain + { + echo "### Pinned" + git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' | grep -E '^\+(pkgver|_commit)=' | sed 's/^+/- /' + } >> "$GITHUB_STEP_SUMMARY" + fi + + # No fallback to GITHUB_TOKEN here: a PR it opened would sit with its + # checks held, and an auto-merge it enabled would land without running + # publish.yml. Better to fail loudly than to pin quietly. + - name: Require the bot App + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + run: | + if [[ -z "$PKGS_BOT_APP_ID" ]]; then + echo "::error::PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY are not set. Create a GitHub App with Contents: write and Pull requests: write, install it on this repository, and store its id and private key as those secrets." + exit 1 + fi + + - name: Mint the bot token + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: app + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.PKGS_BOT_APP_ID }} + private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + + # The PR title names what moved, so the merged history reads like a + # changelog: "Track upstream branches: omarchy-dev 4.0.0.r6520.g1a2b3c4, ...". + - name: Describe the pins + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: describe + run: | + title=$(git diff --unified=0 -- 'pkgbuilds/*/PKGBUILD' \ + | awk '/^\+\+\+ b\/pkgbuilds\//{split($2,p,"/"); pkg=p[3]} /^\+pkgver=/{sub(/^\+pkgver=/,""); printf "%s %s, ", pkg, $0}' \ + | sed 's/, $//') + echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" + + - name: Open or update the tracking PR + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + id: pr + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.app.outputs.token }} + commit-message: ${{ steps.describe.outputs.title }} + title: ${{ steps.describe.outputs.title }} + body: | + Automated pin of packages that follow a moving upstream branch + (`"auto_merge": true` in `.omarchy/package.json`). Each package's + `_commit` now points at the branch tip that has been there for at + least its `min_release_age`. + + This PR auto-merges once the build checks pass. A failing build + leaves it open; the next tracker run replaces it with the newer tip. + branch: auto/track-branches + delete-branch: true + labels: automated + + # Auto-merge, not a direct merge: branch protection still has to see + # `result`, `self-tests` and `build-isolation` green, and this lane + # inherits every rule the reviewed lane has except the human. + - name: Enable auto-merge + if: ${{ !cancelled() && steps.pr.outputs.pull-request-number != '' }} + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + PR: ${{ steps.pr.outputs.pull-request-number }} + run: | + # Idempotent across re-runs of an updated PR: enabling twice errors. + if [[ "$(gh pr view "$PR" -R "${{ github.repository }}" --json autoMergeRequest --jq '.autoMergeRequest != null')" == true ]]; then + echo "auto-merge already enabled on #$PR" + exit 0 + fi + # A PR whose checks all reused existing artifacts can be clean + # before this step runs; GitHub then refuses --auto, so merge it. + gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" \ + || gh pr merge --merge "$PR" -R "${{ github.repository }}" + + - name: Notify Basecamp on failure + if: failure() && env.BASECAMP_CHATBOT_URL != '' + env: + BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} + run: | + curl -s -o /dev/null \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg content \ + "🔴 Branch tracking failed
View run" \ + '{content: $content}')" \ + "$BASECAMP_CHATBOT_URL" diff --git a/README.md b/README.md index 8823595..5e216f7 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,14 @@ The filesystem no longer encodes release policy. Instead: (`OMARCHY_RC_PINS=1`, which `omarchy-release rc` sets) may build it for rc — master's shipped pins can never overwrite an in-flight RC. The dev pair (`omarchy-dev`, `omarchy-settings-dev`) is pinned to `edge` +- packages that follow a moving upstream branch (the dev pair on `quattro`, + `omasnap-git` on `main`) still pin an exact commit in their PKGBUILD. A + `git_branch` upstream watch moves that pin, and `"auto_merge": true` puts the + package on the unattended lane: `track-branches.yml` opens the bump PR every + two hours and auto-merges it once the build checks pass, so a branch tip + reaches the edge channel without anyone clicking. No PKGBUILD may carry an + unpinned git source (`tests/pinned-sources.sh`); a branch that has to be + followed gets a watch, not a `#branch=` fragment - Omarchy owns every checked-in recipe; upstream watches update release metadata without replacing packaging or architecture support - packages can opt out of unscoped builds with `skip_build`; explicit `--package` builds remain available - packages follow direct upstream watches/providers in `.omarchy/package.json`, or a custom `.omarchy/upstream.sh` hook @@ -719,6 +727,7 @@ Fields: - `upstream`: optional direct release watch (see [Upstream watches](docs/upstream-sources.md)), or an existing GitHub, git-tag, npm, or Debian provider. GitHub architecture assets may be a string or an ordered array, and can be combined with disjoint versioned `sources` — see [Sync Upstream Releases](#sync-upstream-releases). Mutually exclusive with `.omarchy/upstream.sh`. - `min_release_age`: optional quarantine for upstream releases (`"24h"`, `"2d"`, or bare seconds). The newest release older than the window ships; anything younger waits, and a release whose age cannot be proven fails the sync. Bypass deliberately with `BYPASS_MIN_RELEASE_AGE=1 bin/sync-upstream `. - `sync`: `false` records an existing manual maintenance hold. Held packages have no upstream watch/provider/hook and are excluded from automatic updates. +- `auto_merge`: optional boolean; defaults to `false`. `true` moves the package's upstream updates from the reviewed 6-hourly sync PR to the unattended lane: `track-branches.yml` opens its bump PR and auto-merges it when CI is green. Meant for packages that follow a moving branch through a `git_branch` watch, where every tip is a release and there is nothing for a reviewer to read. Requires an upstream watch, provider, or hook. - `origin`: optional historical import provenance, with `aur` (package name) and `commit`. It does not control updates. - `release_ring`: optional. `fast` means the package is built directly for stable as well as edge, with the artifacts replicated into rc for parity. Packages without a ring build in edge and reach stable through the pipeline (`bin/repo advance`). - `channels`: optional array bounding where the package may be built (`edge`, `rc`, `stable`). Without the key a package is a member of every channel and follows the default build rules above; `bin/repo advance` refuses to carry a package anywhere it isn't a member. @@ -880,8 +889,17 @@ The repository includes GitHub workflows and systemd services for automated rele #### GitHub Workflows -1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red. +1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes on the reviewed lane. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red. 2. **sync-rebuilds.yml** (Every 6 hours): Bumps pkgrel for packages whose `rebuild_on` dependencies have moved in the official repositories and opens a PR. +3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the newest tip of its watched branch that has sat there for `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. + +The sync PRs are opened with a GitHub App token (`PKGS_BOT_APP_ID` and +`PKGS_BOT_PRIVATE_KEY` secrets; the App needs Contents and Pull requests +write on this repository). A PR opened with the workflow's own `GITHUB_TOKEN` +has its build and test runs held until a maintainer approves them, and an +auto-merge it enabled would land without running the publish workflow. The +reviewed workflows fall back to `GITHUB_TOKEN` when the App is not configured +(and then need that click); the tracker refuses to run without it. To approve builds for an unvouched contributor's PR, apply **`build-approved`**. Until approval, the PR shows **Awaiting build approval** and its required diff --git a/bin/sync-upstream b/bin/sync-upstream index 2e22f9a..a08f184 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -12,6 +12,7 @@ trap 'rm -rf "$TEMP_DIR"' EXIT export UPSTREAM_CACHE_DIR="$TEMP_DIR/watch-cache" SPECIFIC_PACKAGES=() +LANE=all usage() { cat < + Which delivery lane to sync (default: all). Packages marked + "auto_merge": true ride the unattended lane (the branch tracker + opens and auto-merges their PR); everything else is reviewed. + The scheduled workflows each pass their own lane so a moving + branch tip never waits on a vendor release, or the reverse. + Commands: self-test Run the offline fixture tests for release selection, the quarantine backstop, and metadata parsing @@ -65,6 +74,14 @@ while [[ $# -gt 0 ]]; do usage exit 0 ;; + --lane) + LANE="${2:-}" + case "$LANE" in + reviewed|auto-merge|all) ;; + *) print_error "Invalid --lane '$LANE' (expected reviewed, auto-merge, or all)"; exit 1 ;; + esac + shift 2 + ;; --*) print_error "Unknown option: $1" exit 1 @@ -1042,16 +1059,82 @@ if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 && "${SPECIFIC_PACKAGES[0]}" == "self-test" exit 0 fi +# Packages that pin the same upstream branch move together or not at all. +# The watch reads one shared clone per run, so they only disagree when one +# package's update failed after the tip was chosen (a checksum fetch, say). +# Leaving the other one advanced would ship omarchy-dev and +# omarchy-settings-dev from different commits, which is exactly the skew the +# release pair's lockstep guard exists to prevent. Restore the packages that +# moved and count the group as failed; the next run tries again. +declare -A BEFORE_SYNC=() + +snapshot_package() { + local package="$1" pkgbuild="$PKGBUILDS_DIR/$1/PKGBUILD" + [[ -f "$pkgbuild" ]] || return 0 + mkdir -p "$TEMP_DIR/before" + cp "$pkgbuild" "$TEMP_DIR/before/$package" + BEFORE_SYNC["$package"]=1 +} + +branch_watch_key() { + local package_dir="$1" + jq -r ' + (.upstream.watch? | objects | select(has("git_branch"))) + | "\(.git_branch)#\(.branch)" + ' "$package_dir/.omarchy/package.json" 2>/dev/null +} + +enforce_branch_lockstep() { + local package key + declare -A groups=() + for package in "${!BEFORE_SYNC[@]}"; do + key=$(branch_watch_key "$PKGBUILDS_DIR/$package") + [[ -n "$key" ]] || continue + groups["$key"]+="$package " + done + for key in "${!groups[@]}"; do + local members commits + read -r -a members <<<"${groups[$key]}" + (( ${#members[@]} > 1 )) || continue + commits=$(for package in "${members[@]}"; do + grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" + done | sort -u | grep -c .) + (( commits > 1 )) || continue + print_error "Lockstep violation on $key: ${members[*]} pin different commits; restoring all of them" + for package in "${members[@]}"; do + if ! cmp -s "$TEMP_DIR/before/$package" "$PKGBUILDS_DIR/$package/PKGBUILD"; then + cp "$TEMP_DIR/before/$package" "$PKGBUILDS_DIR/$package/PKGBUILD" + # Not ((--UPDATED)): an arithmetic command that evaluates to zero + # returns 1, and under errexit that would end the run right here. + UPDATED=$((UPDATED > 0 ? UPDATED - 1 : 0)) + fi + done + ((++FAILED)) + done +} + +sync_in_lane() { + local package="$1" package_dir="$PKGBUILDS_DIR/$1" + if [[ -d "$package_dir" ]] && ! package_in_lane "$package_dir" "$LANE"; then + print_info "Skipping $package: not in the $LANE lane" + ((++SKIPPED)) + return 0 + fi + snapshot_package "$package" + sync_package "$package" +} + if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then SPECIFIC_MODE=true for package in "${SPECIFIC_PACKAGES[@]}"; do - sync_package "$package" + sync_in_lane "$package" done else while IFS= read -r package; do - sync_package "$package" + sync_in_lane "$package" done < <(packages_for_upstream_sync) fi +enforce_branch_lockstep echo "" if [[ $FAILED -gt 0 ]]; then diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index 9ef208f..79cff76 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -48,8 +48,40 @@ pkgrel to 1, but the complete epoch:pkgver-pkgrel must still increase. GitHub releases exclude drafts and prereleases unless `allow_prerelease` is true. Existing `min_release_age` policies apply: a feed without a verifiable publication -time cannot bypass a configured hold. Git branch watches derive a commit count -and date from the actual branch history and write an immutable source pin. +time cannot bypass a configured hold. + +## Branch watches + +A `git_branch` watch treats every commit on a branch as a release and writes an +immutable pin (`"_commit": "{commit}"`) so the recipe never carries a moving +`#branch=` source; `tests/pinned-sources.sh` enforces that. The clone is bare, +blobless and single-branch, read only with git, and shared by every package +that watches the same branch in one run, so two recipes pinned from it always +see the same commit. Values available to `version`: + +- `{date}` (default), `{count}` (commits on the branch), `{commit}` + (`{commit:.7}` for the short form) +- with `tag_pattern` (a regular expression with a named `version` group, + matched against whole tags): `{tag}`, `{version}` from that tag, and + `{distance}`, the number of commits past it. Only tags in the pinned + commit's own history count, so a release cut on another branch is ignored. + +`{version}.r{distance}.g{commit:.7}` gives `1.21.0.r15.gabc1234`, which pacman +orders above the `1.21.0` release it follows and below `1.21.1`; `omasnap-git` +uses it. The Omarchy dev pair uses `{version}.r{count}.g{commit:.7}` instead +because its published history counted every commit and the number must never +go down. + +`min_release_age` on a branch watch selects the newest commit that has been on +the branch for at least that long, so a burst of pushes builds once after it +settles rather than once per push. `BYPASS_MIN_RELEASE_AGE=1` takes the tip. + +Packages marked `"auto_merge": true` ride the unattended lane +(`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened +and auto-merged as soon as the build checks pass. `bin/sync-upstream --lane +reviewed|auto-merge|all` selects a lane; the scheduled workflows each pass their +own. Packages that pin the same branch move in lockstep: if one of them fails +to update, the run restores the others and reports the group as failed. Checksums retain their algorithms (SHA256, SHA512, BLAKE2, etc.) and source order. Changed git sources are hashed with makepkg's git-archive convention. Unchanged @@ -131,6 +163,8 @@ in `origin` and has no effect on release selection. | `localsend` | github | [localsend/localsend](https://github.com/localsend/localsend) | | `localsend-bin` | github | [localsend/localsend](https://github.com/localsend/localsend) | | `macbook12-spi-driver-dkms` | git_branch | [https://github.com/marc-git/macbook12-spi-driver.git](https://github.com/marc-git/macbook12-spi-driver.git) | +| `omarchy-dev`, `omarchy-settings-dev` | git_branch (auto-merge) | [https://github.com/basecamp/omarchy.git](https://github.com/basecamp/omarchy.git) `quattro` | +| `omasnap-git` | git_branch (auto-merge) | [https://github.com/omacom/omasnap.git](https://github.com/omacom/omasnap.git) `main` | | `makima-bin` | github | [cyber-sushi/makima](https://github.com/cyber-sushi/makima) | | `minecraft-launcher` | archive | [https://launcher.mojang.com/download/Minecraft.deb](https://launcher.mojang.com/download/Minecraft.deb) | | `nautilus-dropbox` | github | [dropbox/nautilus-dropbox](https://github.com/dropbox/nautilus-dropbox) | diff --git a/helpers/package-metadata.sh b/helpers/package-metadata.sh index ce46e92..23288bc 100644 --- a/helpers/package-metadata.sh +++ b/helpers/package-metadata.sh @@ -13,6 +13,7 @@ # { "source": "local", "channels": ["edge"] } # { "source": "local", "channels": ["edge", "rc", "stable"] } # { "source": "local", "min_release_age": "24h" } +# { "source": "local", "auto_merge": true, "upstream": { "watch": { "git_branch": "...", "branch": "main" } } } # { "source": "local", "upstream": { "github": "owner/repo", "checksums": "SHASUMS256.txt", "assets": { "x86_64": ["name-{tag}-x64.tar.xz"] } } } # { "source": "local", "upstream": { "github": "owner/repo", "digests": true, "assets": { "x86_64": "name-{tag}-x64.tar.xz" } } } # { "source": "local", "upstream": { "git_tags": "https://example/repo.git", "tag_pattern": "v{pkgver}", "sources": { "any": ["https://example/archive/{tag}.tar.gz"] } } } @@ -327,6 +328,31 @@ packages_for_upstream_sync() { done } +# Upstream updates travel in one of two lanes. The reviewed lane is the +# 6-hourly sync PR a maintainer reads before merging. A package that marks +# "auto_merge": true rides the unattended lane instead: its bump PR is opened +# and auto-merged by the branch tracker as soon as CI is green, which is how a +# package that follows a moving branch (omarchy-dev, omasnap-git) gets rebuilt +# without anyone clicking. The lanes are disjoint so a branch tip can never +# hold up a reviewed vendor release, or the other way round. +package_auto_merge() { + local pkgdir="$1" metadata + metadata=$(metadata_file_for_dir "$pkgdir") + [[ -f "$metadata" ]] || return 1 + [[ "$(jq -r 'if has("auto_merge") then .auto_merge else false end' "$metadata")" == "true" ]] +} + +# package_in_lane +package_in_lane() { + local pkgdir="$1" lane="$2" + case "$lane" in + all | "") return 0 ;; + auto-merge) package_auto_merge "$pkgdir" ;; + reviewed) ! package_auto_merge "$pkgdir" ;; + *) echo "invalid lane: $lane (expected reviewed, auto-merge, or all)" >&2; return 2 ;; + esac +} + # Packages that must be rebuilt when a dependency they link against changes, # even though nothing in their own source moved. `rebuild_on` names those # dependencies; `rebuilt_against` records the versions the checked-in pkgrel was @@ -514,6 +540,15 @@ validate_package_metadata() { return 1 fi + if ! jq -e 'if has("auto_merge") | not then true else (.auto_merge | type) == "boolean" end' "$metadata" >/dev/null; then + echo "invalid auto_merge for $(basename "$pkgdir"): must be boolean" + return 1 + fi + if package_auto_merge "$pkgdir" && ! package_has_upstream_provider "$pkgdir" && ! package_has_upstream_hook "$pkgdir"; then + echo "invalid auto_merge for $(basename "$pkgdir"): only an upstream watch, provider, or hook can be auto-merged" + return 1 + fi + # `has` rather than `// {}`: jq's // treats false as absent, which would # let "upstream": false slip through as an empty declaration. if ! jq -e ' diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index 6e17b97..05828f4 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -82,7 +82,7 @@ def validate(watch): allowed = PROVIDERS | {"pattern", "path", "package", "branch", "variables", "fields", "submodules", "allow_prerelease", "unescape_json", "filenames", "sequence", "version", "revision", "revision_variable", - "mutable_sources", "member", "dist_tag"} + "mutable_sources", "member", "dist_tag", "tag_pattern"} if watch.keys() - allowed: raise ValueError(f"unknown watch fields: {sorted(watch.keys() - allowed)}") value = watch[provider] @@ -113,6 +113,18 @@ def validate(watch): if not isinstance(branch, str) or not branch or branch.startswith("-"): raise ValueError("git branch watch needs an explicit branch") run(["git", "check-ref-format", "refs/heads/" + branch]) + # A branch watch whose version template names a tag needs to know + # which tags count as releases; anything else is an untagged branch. + if "tag_pattern" in watch: + if not isinstance(watch["tag_pattern"], str) or not watch["tag_pattern"]: + raise ValueError("watch.tag_pattern must be a regular expression string") + if "version" not in re.compile(watch["tag_pattern"]).groupindex: + raise ValueError("tag_pattern needs a named version group") + template = watch.get("version", "{version}") + if any(field in template for field in ("{tag", "{distance")) and "tag_pattern" not in watch: + raise ValueError("a version built from {tag}/{distance} needs a tag_pattern") + elif "tag_pattern" in watch: + raise ValueError("tag_pattern only applies to git_branch watches") for field in ("variables", "submodules", "fields"): mapping = watch.get(field, {}) if not isinstance(mapping, dict): @@ -172,7 +184,61 @@ def matches(watch, text, extra=None, full=False): yield candidate(watch, {**(extra or {}), **match.groupdict()}) -def discover(watch, fetch): +def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): + """Describe the newest commit on an upstream branch that has sat there for + at least min_age seconds (the branch analogue of "the newest release older + than the window ships"): commit, total count, date, and with a tag_pattern + the newest release tag reachable from it plus the distance from that tag, + so a branch build can be versioned .r.g, above the release it + follows and below the next one, the way a pkgver() function would. + + One blobless single-branch clone per (url, branch) per run, shared by + every package that tracks it, so two recipes pinned from one clone always + see the same commit. The clone is read with git only; nothing in it runs. + Returns None when every commit is younger than the window. + """ + https(url) + key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest() + work = Path(cache) / f"{key}.branch.git" + if not work.exists(): + scratch = work.with_name(f"{work.name}.{os.getpid()}.tmp") + subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True) + scratch.replace(work) + git = ["git", "-C", str(work)] + selector = ["HEAD"] + if min_age: + cutoff = (now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(seconds=min_age) + selector = ["-1", f"--before={cutoff.isoformat()}", "HEAD"] + commit = run([*git, "rev-list", *selector], text=True).split()[:1] + if not commit: + return None + commit = commit[0] + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("branch tip is not a commit") + count = run([*git, "rev-list", "--count", commit], text=True).strip() + date = run([*git, "show", "-s", "--format=%cs", commit], text=True).strip().replace("-", "") + timestamp = run([*git, "show", "-s", "--format=%cI", commit], text=True).strip() + values = {"version": date, "date": date, "count": count, "commit": commit, "published_at": timestamp} + if tag_pattern: + pattern = re.compile(tag_pattern) + best = None + # Only tags in this commit's history count; a release cut on another + # branch is not something this branch is "past". + for tag in run([*git, "tag", "--merged", commit], text=True).split(): + match = pattern.fullmatch(tag) + if not match: + continue + version = match.group("version") + if best is None or vercmp(version, best[0]) > 0: + best = (version, tag) + if best is None: + raise ValueError(f"no tag on {branch} matches {tag_pattern}") + distance = run([*git, "rev-list", "--count", f"{best[1]}..{commit}"], text=True).strip() + values.update({"tag": best[1], "version": best[0], "distance": distance}) + return values + + +def discover(watch, fetch, min_age=0): provider = validate(watch) feed = watch[provider] results = [] @@ -199,13 +265,10 @@ def discover(watch, fetch): for tag, commit in tags.items(): results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True)) elif provider == "git_branch": - with tempfile.TemporaryDirectory(prefix="upstream-git-") as work: - subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", watch["branch"], feed, work], check=True) - commit = run(["git", "-C", work, "rev-parse", "HEAD"], text=True).strip() - count = run(["git", "-C", work, "rev-list", "--count", "HEAD"], text=True).strip() - date = run(["git", "-C", work, "show", "-s", "--format=%cs", "HEAD"], text=True).strip().replace("-", "") - timestamp = run(["git", "-C", work, "show", "-s", "--format=%cI", "HEAD"], text=True).strip() - results.append(candidate(watch, {"version": date, "date": date, "count": count, "commit": commit, "published_at": timestamp})) + tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age) + if tip is None: + return [] # nothing has settled for min_age yet: wait, not an error + results.append(candidate(watch, tip)) elif provider == "npm": data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe="")) version = data["dist-tags"][watch.get("dist_tag", "latest")] @@ -479,7 +542,8 @@ def sync(package, fetch, min_age=0, check=False): path = package / "PKGBUILD" original = path.read_text() before = read_recipe(path) - release = select_release(discover(watch, fetch), min_age, bypass=os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1") + bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1" + release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass) if release is None: return {"status": "skipped", "reason": "minimum release age"} current = scalar(before, "pkgver") diff --git a/pkgbuilds/omarchy-dev/.omarchy/package.json b/pkgbuilds/omarchy-dev/.omarchy/package.json index 9adf372..64dc205 100644 --- a/pkgbuilds/omarchy-dev/.omarchy/package.json +++ b/pkgbuilds/omarchy-dev/.omarchy/package.json @@ -1 +1,17 @@ -{ "source": "local", "channels": ["edge"] } +{ + "source": "local", + "channels": ["edge"], + "auto_merge": true, + "min_release_age": "30m", + "upstream": { + "watch": { + "git_branch": "https://github.com/basecamp/omarchy.git", + "branch": "quattro", + "tag_pattern": "v(?P[0-9]+\\.[0-9]+\\.[0-9]+)", + "version": "{version}.r{count}.g{commit:.7}", + "variables": { + "_commit": "{commit}" + } + } + } +} diff --git a/pkgbuilds/omarchy-dev/PKGBUILD b/pkgbuilds/omarchy-dev/PKGBUILD index 58630ed..c2e70ea 100644 --- a/pkgbuilds/omarchy-dev/PKGBUILD +++ b/pkgbuilds/omarchy-dev/PKGBUILD @@ -1,9 +1,13 @@ # Maintainer: Ryan Hughes pkgname='omarchy-dev' -pkgver=4.0.0.r847.g4de185b +pkgver=4.0.0.r6514.gee8ebf6 pkgrel=1 -_pkgver_base=4.0.0 -_pkgver_base_tag=v3.8.2 +# Pinned by the upstream watch in .omarchy/package.json (bin/sync-upstream): +# every quattro tip becomes a commit pin here, so the package is versioned, +# checksummed and built exactly like a release, just more often. The r-number +# is the branch's total commit count, not the distance from the last tag: the +# published history used the total, and pacman must never see it go down. +_commit=ee8ebf615dcbf9e1a126b22a894f907d6795a9a5 pkgdesc='Beautiful, modern, and opinionated Arch Linux by DHH (quattro branch tip)' # The payload is architecture-independent, but the dependency set is not: the # boot stack differs per architecture (see depends_x86_64 / depends_aarch64), @@ -73,30 +77,16 @@ makedepends=( 'git' ) -# Source: quattro branch tip by default; set OMARCHY_SRC=/path/to/checkout -# to build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX). +# Source: the pinned quattro commit. Set OMARCHY_SRC=/path/to/checkout to +# build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX); +# the arrays are emptied below so nothing is downloaded in that case. +source=("omarchy::git+https://github.com/basecamp/omarchy.git#commit=${_commit}") +sha256sums=('4f55a96e32143c766f925ec7f721a374c32bd20e66f9a081f5dbe0772c609ecd') if [[ -n "${OMARCHY_SRC:-}" ]]; then source=() sha256sums=() -else - source=("omarchy::git+https://github.com/basecamp/omarchy.git#branch=quattro") - sha256sums=('SKIP') fi -pkgver() { - cd "$srcdir/omarchy" - - local commit_count commit_hash - if git rev-parse --verify "${_pkgver_base_tag}^{commit}" >/dev/null 2>&1; then - commit_count=$(git rev-list --count "${_pkgver_base_tag}..HEAD") - else - commit_count=$(git rev-list --count HEAD) - fi - commit_hash=$(git rev-parse --short=7 HEAD) - - printf "%s.r%s.g%s" "$_pkgver_base" "$commit_count" "$commit_hash" -} - prepare() { if [[ -n "${OMARCHY_SRC:-}" ]]; then rm -rf "$srcdir/omarchy" diff --git a/pkgbuilds/omarchy-settings-dev/.omarchy/package.json b/pkgbuilds/omarchy-settings-dev/.omarchy/package.json index 9adf372..64dc205 100644 --- a/pkgbuilds/omarchy-settings-dev/.omarchy/package.json +++ b/pkgbuilds/omarchy-settings-dev/.omarchy/package.json @@ -1 +1,17 @@ -{ "source": "local", "channels": ["edge"] } +{ + "source": "local", + "channels": ["edge"], + "auto_merge": true, + "min_release_age": "30m", + "upstream": { + "watch": { + "git_branch": "https://github.com/basecamp/omarchy.git", + "branch": "quattro", + "tag_pattern": "v(?P[0-9]+\\.[0-9]+\\.[0-9]+)", + "version": "{version}.r{count}.g{commit:.7}", + "variables": { + "_commit": "{commit}" + } + } + } +} diff --git a/pkgbuilds/omarchy-settings-dev/PKGBUILD b/pkgbuilds/omarchy-settings-dev/PKGBUILD index 1d4dce0..2fb1016 100644 --- a/pkgbuilds/omarchy-settings-dev/PKGBUILD +++ b/pkgbuilds/omarchy-settings-dev/PKGBUILD @@ -1,9 +1,13 @@ # Maintainer: Ryan Hughes pkgname='omarchy-settings-dev' -pkgver=4.0.0.r847.g4de185b -pkgrel=2 -_pkgver_base=4.0.0 -_pkgver_base_tag=v3.8.2 +pkgver=4.0.0.r6514.gee8ebf6 +pkgrel=1 +# Pinned by the upstream watch in .omarchy/package.json (bin/sync-upstream): +# every quattro tip becomes a commit pin here, so the package is versioned, +# checksummed and built exactly like a release, just more often. The r-number +# is the branch's total commit count, not the distance from the last tag: the +# published history used the total, and pacman must never see it go down. +_commit=ee8ebf615dcbf9e1a126b22a894f907d6795a9a5 pkgdesc='Omarchy user defaults, /etc/skel content, fonts, plymouth theme, and support helpers (quattro branch tip)' # Arch-specific because the shipped /etc tree is not the same on every # architecture: the Limine, mkinitcpio, zram and oomd drop-ins belong to the @@ -109,30 +113,16 @@ _etc_override_paths=( 'etc/plymouth/plymouthd.conf' ) -# Source: quattro branch tip by default; set OMARCHY_SRC=/path/to/checkout -# to build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX). +# Source: the pinned quattro commit. Set OMARCHY_SRC=/path/to/checkout to +# build from a local source tree instead (mirrors omarchy-zsh/omarchy-fish DX); +# the arrays are emptied below so nothing is downloaded in that case. +source=("omarchy::git+https://github.com/basecamp/omarchy.git#commit=${_commit}") +sha256sums=('4f55a96e32143c766f925ec7f721a374c32bd20e66f9a081f5dbe0772c609ecd') if [[ -n "${OMARCHY_SRC:-}" ]]; then source=() sha256sums=() -else - source=("omarchy::git+https://github.com/basecamp/omarchy.git#branch=quattro") - sha256sums=('SKIP') fi -pkgver() { - cd "$srcdir/omarchy" - - local commit_count commit_hash - if git rev-parse --verify "${_pkgver_base_tag}^{commit}" >/dev/null 2>&1; then - commit_count=$(git rev-list --count "${_pkgver_base_tag}..HEAD") - else - commit_count=$(git rev-list --count HEAD) - fi - commit_hash=$(git rev-parse --short=7 HEAD) - - printf "%s.r%s.g%s" "$_pkgver_base" "$commit_count" "$commit_hash" -} - prepare() { if [[ -n "${OMARCHY_SRC:-}" ]]; then rm -rf "$srcdir/omarchy" diff --git a/pkgbuilds/omasnap-git/.omarchy/package.json b/pkgbuilds/omasnap-git/.omarchy/package.json new file mode 100644 index 0000000..4575123 --- /dev/null +++ b/pkgbuilds/omasnap-git/.omarchy/package.json @@ -0,0 +1,17 @@ +{ + "source": "local", + "channels": ["edge"], + "auto_merge": true, + "min_release_age": "30m", + "upstream": { + "watch": { + "git_branch": "https://github.com/omacom/omasnap.git", + "branch": "main", + "tag_pattern": "v(?P[0-9]+\\.[0-9]+\\.[0-9]+)", + "version": "{version}.r{distance}.g{commit:.7}", + "variables": { + "_commit": "{commit}" + } + } + } +} diff --git a/pkgbuilds/omasnap-git/PKGBUILD b/pkgbuilds/omasnap-git/PKGBUILD new file mode 100644 index 0000000..3c9ea2d --- /dev/null +++ b/pkgbuilds/omasnap-git/PKGBUILD @@ -0,0 +1,72 @@ +# Maintainer: Ryan Hughes +# The main-branch build of omasnap. Pinned by the upstream watch in +# .omarchy/package.json (bin/sync-upstream): every main tip becomes a commit +# pin here, versioned .r.g so it sorts above the +# tagged release it follows and below the next one. + +pkgname=omasnap-git +pkgver=1.21.0.r15.geb22bfe +pkgrel=1 +_commit=eb22bfe5b79a2235f9bf5a8ffd026bc882969c1e +pkgdesc="Native Wayland screenshot and annotation overlay for Hyprland (main branch)" +arch=('x86_64' 'aarch64') +url="https://github.com/omacom/omasnap" +license=('MIT' 'OFL-1.1') +depends=( + 'hyprland' + 'layer-shell-qt' + 'qt6-base' + 'tesseract' + 'tesseract-data-eng' + 'wayland' + 'wl-clipboard' +) +makedepends=( + 'cmake' + 'git' + 'ninja' + 'pkgconf' + 'wayland-protocols' +) +provides=('omasnap') +conflicts=('omasnap') +options=('!debug') + +source=("omasnap::git+$url.git#commit=${_commit}") +sha256sums=('c8717bae97faa3798cce25038d42429185f832cd47b09f20c232f684c721952a') + +build() { + cmake -S omasnap -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr + cmake --build build --parallel +} + +check() { + # The smoke suite fsyncs its working documents under /tmp. On the CI + # droplets the build leaves about a gigabyte of dirty pages, and the + # flush that starts a few seconds into the suite makes those fsyncs stall + # long enough to overrun the suite's 5-second settle windows. Flush first. + local runtime_dir status started + started=$(date +%s%N); sync + echo "flushed dirty pages in $(( ($(date +%s%N) - started) / 1000000 )) ms" + runtime_dir=$(mktemp -d /dev/shm/omasnap-runtime.XXXXXX) + if QT_QPA_PLATFORM=offscreen QT_FORCE_STDERR_LOGGING=1 \ + XDG_RUNTIME_DIR="$runtime_dir" \ + ./build/omasnap-smoke "$srcdir/omasnap-smoke-output"; then + status=0 + else + status=$? + echo "omasnap-smoke exited with status $status" >&2 + fi + rm -r -- "$runtime_dir" + return "$status" +} + +package() { + cmake --install build --prefix "$pkgdir/usr" + install -Dm644 omasnap/README.md \ + "$pkgdir/usr/share/doc/omasnap/README.md" + install -Dm644 omasnap/LICENSE \ + "$pkgdir/usr/share/licenses/omasnap/LICENSE" +} diff --git a/tests/pinned-sources.sh b/tests/pinned-sources.sh new file mode 100755 index 0000000..6392089 --- /dev/null +++ b/tests/pinned-sources.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Every git source in the repository names an immutable commit or a tag. +# +# A source that follows a branch ("#branch=quattro", or no fragment at all) +# produces a package whose contents depend on when it was built, and nothing +# in this repository changes when that branch moves, so the CI publish path, +# which builds what a merge touched, never rebuilds it. Packages that need to +# follow a branch declare a git_branch upstream watch instead, and the tracker +# turns each new tip into a commit pin here (docs/upstream-sources.md). +set -euo pipefail +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +PKGBUILDS_DIR=${PKGBUILDS_DIR:-$BUILD_ROOT/pkgbuilds} + +failures=0 +checked=0 +for pkgdir in "$PKGBUILDS_DIR"/*/; do + [[ -f "$pkgdir/PKGBUILD" ]] || continue + package=$(basename "$pkgdir") + for arch in x86_64 aarch64; do + # Sourced the way the build tooling reads recipes: CARCH set, the local + # source override unset, so conditional and arch-suffixed arrays count. + sources=$(cd "$pkgdir" && env -u OMARCHY_SRC CARCH="$arch" bash -c ' + source PKGBUILD >/dev/null 2>&1 + printf "%s\n" "${source[@]}" "${source_x86_64[@]}" "${source_aarch64[@]}"' 2>/dev/null) || { + echo "FAIL: $package: PKGBUILD could not be sourced for $arch" + failures=$((failures + 1)) + continue + } + while IFS= read -r entry; do + [[ -n "$entry" ]] || continue + url="${entry#*::}" + [[ "$url" == git+* ]] || continue + checked=$((checked + 1)) + case "$url" in + *'#commit='*|*'#tag='*) ;; + *) + echo "FAIL: $package ($arch): git source is not pinned to a commit or tag: $url" + failures=$((failures + 1)) + ;; + esac + done <<<"$sources" + done +done + +if ((failures)); then + echo "$failures unpinned git source(s). Pin with #commit= (and a git_branch upstream watch to move the pin), or #tag= with a checksum." + exit 1 +fi +echo "PASS: $checked git source(s) across pkgbuilds/ are pinned to a commit or tag" diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index bb9c8dc..4861e16 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -178,6 +178,85 @@ b2sums=('old' 'local-b2') expected = subprocess.check_output(['git', '-c', 'core.abbrev=no', '-C', str(repo), 'archive', '--format', 'tar', 'v1.0']) self.assertEqual(archive.read_bytes(), expected) + def branch_fixture(self, fresh_tip=False): + """An upstream with two release tags and commits past the newest one, + all committed years ago; with fresh_tip, one more commit dated now.""" + repo = self.root / 'branch-upstream' + repo.mkdir() + git = ['git', '-C', str(repo), '-c', 'user.name=Test', '-c', 'user.email=test@example.test'] + old = {**os.environ, 'GIT_COMMITTER_DATE': '2020-01-01T00:00:00+00:00', 'GIT_AUTHOR_DATE': '2020-01-01T00:00:00+00:00'} + subprocess.run(['git', 'init', '-q', '-b', 'main', str(repo)], check=True) + shas = [] + def commit(index, env): + (repo / 'source').write_text(f'revision {index}') + subprocess.run([*git, 'add', '.'], check=True) + subprocess.run([*git, 'commit', '-qm', f'commit {index}'], env=env, check=True) + shas.append(subprocess.check_output([*git, 'rev-parse', 'HEAD'], text=True).strip()) + for index, tag in enumerate([None, 'v1.0.0', 'v1.1.0', None, None]): + commit(index, old) + if tag: + subprocess.run([*git, 'tag', tag], check=True) + # A newer release tagged on another branch is not something main is "past". + subprocess.run([*git, 'checkout', '-q', '-b', 'hotfix', shas[1]], check=True) + (repo / 'hotfix').write_text('x') + subprocess.run([*git, 'add', '.'], check=True) + subprocess.run([*git, 'commit', '-qm', 'hotfix'], env=old, check=True) + subprocess.run([*git, 'tag', 'v9.9.9'], check=True) + subprocess.run([*git, 'checkout', '-q', 'main'], check=True) + if fresh_tip: + commit(len(shas), os.environ) + return repo, shas + + def redirect_clone(self, repo): + command = w.subprocess.run + def redirect(args, **kwargs): + if 'clone' in args: + args = [f'file://{repo}' if arg == 'https://example.test/tool.git' else arg for arg in args] + return command(args, **kwargs) + return patch.object(w.subprocess, 'run', side_effect=redirect) + + def test_git_branch_versions_from_reachable_tag_and_shares_one_clone(self): + repo, shas = self.branch_fixture() + with self.redirect_clone(repo): + tip = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P[0-9.]+)', self.fetch.cache) + again = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache) + self.assertEqual((tip['commit'], tip['tag'], tip['version'], tip['distance'], tip['count']), (shas[-1], 'v1.1.0', '1.1.0', '2', '5')) + self.assertEqual(again['commit'], shas[-1]) + self.assertEqual(len(list(self.fetch.cache.glob('*.branch.git'))), 1, 'one clone per branch per run') + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main', 'tag_pattern': r'v(?P[0-9.]+)', + 'version': '{version}.r{distance}.g{commit:.7}', 'variables': {'_commit': '{commit}'}} + pkgver = w.candidate(watch, tip)['pkgver'] + self.assertEqual(pkgver, f'1.1.0.r2.g{shas[-1][:7]}') + self.assertEqual(w.vercmp(pkgver, '1.1.0'), 1) + self.assertEqual(w.vercmp(pkgver, '1.1.1'), -1) + with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'): + w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P[0-9.]+)', self.root / 'other-cache') + + def test_git_branch_min_age_selects_the_newest_settled_commit(self): + repo, shas = self.branch_fixture(fresh_tip=True) + with self.redirect_clone(repo): + tip = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache) + settled = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P[0-9.]+)', self.fetch.cache, min_age=3600) + nothing = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache, min_age=10 ** 9) + self.assertEqual(tip['commit'], shas[-1], 'no window: the fresh tip') + self.assertEqual((settled['commit'], settled['distance'], settled['count']), (shas[-2], '2', '5'), 'one hour window: the commit before it') + self.assertIsNone(nothing, 'a window older than every commit selects nothing') + watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + with self.redirect_clone(repo): + self.assertEqual(w.discover(watch, self.fetch, min_age=10 ** 9), []) + self.assertEqual(w.discover(watch, self.fetch, min_age=3600)[0]['values']['commit'], shas[-2]) + + def test_git_branch_tag_template_requires_tag_pattern(self): + base = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} + w.validate({**base, 'version': '{date}.r{count}'}) + w.validate({**base, 'tag_pattern': r'v(?P[0-9.]+)', 'version': '{version}.r{distance}.g{commit:.7}'}) + for extra in [{'version': '{version}.r{distance}'}, {'version': '{tag}'}, {'tag_pattern': 'v[0-9.]+'}, + {'tag_pattern': 7}, {'tag_pattern': ''}]: + with self.subTest(extra=extra), self.assertRaises(ValueError): + w.validate({**base, **extra}) + with self.assertRaisesRegex(ValueError, 'only applies'): + w.validate({'github': 'owner/tool', 'pattern': r'v(?P[0-9.]+)', 'tag_pattern': r'v(?P[0-9.]+)'}) + def test_invalid_optional_metadata_fails_validation(self): valid = {'github': 'owner/tool', 'pattern': r'v(?P[0-9.]+)'} for extra in [{'variables': []}, {'fields': {'commit': 3}}, {'submodules': {'pkgver': 'libs/common'}},