diff --git a/.github/workflows/sync-rebuilds.yml b/.github/workflows/sync-rebuilds.yml index a921e83..4d8af04 100644 --- a/.github/workflows/sync-rebuilds.yml +++ b/.github/workflows/sync-rebuilds.yml @@ -68,27 +68,11 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - # App token rather than GITHUB_TOKEN so the PR's build and test runs - # start without a maintainer approving them (see sync-upstream.yml). - - name: Mint the bot token - if: steps.changes.outputs.has_changes == 'true' - id: app - env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - # Without the App configured this falls back to GITHUB_TOKEN below, - # which still opens the PR; a maintainer then has to approve its - # workflow runs by hand, as before. - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.PKGS_BOT_APP_ID }} - private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} - continue-on-error: true - - name: Create Pull Request if: steps.changes.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + uses: peter-evans/create-pull-request@v7 with: - token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} commit-message: 'chore: rebuild against updated dependencies' title: 'chore: rebuild against updated dependencies' body: | diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml index 971d461..5eaa588 100644 --- a/.github/workflows/sync-upstream.yml +++ b/.github/workflows/sync-upstream.yml @@ -72,30 +72,11 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - # A PR opened with GITHUB_TOKEN gets its build and test runs held - # until a maintainer clicks "Approve workflows to run"; one opened by - # the App builds on its own, so the reviewer sees a green (or red) PR - # instead of a pending one. The App only opens the PR: merging stays - # a human decision in this lane. - - name: Mint the bot token - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - id: app - env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - # Without the App configured this falls back to GITHUB_TOKEN below, - # which still opens the PR; a maintainer then has to approve its - # workflow runs by hand, as before. - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - app-id: ${{ secrets.PKGS_BOT_APP_ID }} - private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} - continue-on-error: true - - name: Create Pull Request if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + uses: peter-evans/create-pull-request@v7 with: - token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} commit-message: 'chore: sync upstream releases' title: 'chore: sync upstream releases' body: | diff --git a/.github/workflows/track-branches.yml b/.github/workflows/track-branches.yml index 5413855..cf063b2 100644 --- a/.github/workflows/track-branches.yml +++ b/.github/workflows/track-branches.yml @@ -39,6 +39,16 @@ jobs: contents: read steps: + - name: Require the bot App + env: + PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} + PKGS_BOT_PRIVATE_KEY: ${{ secrets.PKGS_BOT_PRIVATE_KEY }} + run: | + if [[ -z "$PKGS_BOT_APP_ID" || -z "$PKGS_BOT_PRIVATE_KEY" ]]; then + echo "::error::Set PKGS_BOT_APP_ID and PKGS_BOT_PRIVATE_KEY for a GitHub App installed on this repository with Contents: write and Pull requests: write." + exit 1 + fi + - name: Checkout repository uses: actions/checkout@v4 with: @@ -93,19 +103,6 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" fi - # No fallback to GITHUB_TOKEN here: a PR it opened would sit with its - # checks held, and an auto-merge it enabled would land without running - # publish.yml. Better to fail loudly than to pin quietly. - - name: Require the bot App - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} - env: - PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }} - run: | - if [[ -z "$PKGS_BOT_APP_ID" ]]; then - echo "::error::PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY are not set. Create a GitHub App with Contents: write and Pull requests: write, install it on this repository, and store its id and private key as those secrets." - exit 1 - fi - - name: Mint the bot token if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} id: app @@ -126,7 +123,7 @@ jobs: echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT" - name: Open or update the tracking PR - if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }} + if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' && steps.app.outcome == 'success' }} id: pr uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: @@ -136,8 +133,8 @@ jobs: body: | Automated pin of packages that follow a moving upstream branch (`"auto_merge": true` in `.omarchy/package.json`). Each package's - `_commit` now points at the branch tip that has been there for at - least its `min_release_age`. + `_commit` now points at the branch tip. Fresh tips wait until + their commit timestamp is at least `min_release_age` old. This PR auto-merges once the build checks pass. A failing build leaves it open; the next tracker run replaces it with the newer tip. @@ -159,10 +156,7 @@ jobs: echo "auto-merge already enabled on #$PR" exit 0 fi - # A PR whose checks all reused existing artifacts can be clean - # before this step runs; GitHub then refuses --auto, so merge it. - gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" \ - || gh pr merge --merge "$PR" -R "${{ github.repository }}" + gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" - name: Notify Basecamp on failure if: failure() && env.BASECAMP_CHATBOT_URL != '' diff --git a/README.md b/README.md index 5e216f7..9742113 100644 --- a/README.md +++ b/README.md @@ -891,15 +891,15 @@ The repository includes GitHub workflows and systemd services for automated rele 1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes on the reviewed lane. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red. 2. **sync-rebuilds.yml** (Every 6 hours): Bumps pkgrel for packages whose `rebuild_on` dependencies have moved in the official repositories and opens a PR. -3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the newest tip of its watched branch that has sat there for `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. +3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the tip of its watched branch once its commit timestamp clears `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all, including targeted syncs. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it. -The sync PRs are opened with a GitHub App token (`PKGS_BOT_APP_ID` and +The tracking PR is opened with a GitHub App token (`PKGS_BOT_APP_ID` and `PKGS_BOT_PRIVATE_KEY` secrets; the App needs Contents and Pull requests write on this repository). A PR opened with the workflow's own `GITHUB_TOKEN` has its build and test runs held until a maintainer approves them, and an auto-merge it enabled would land without running the publish workflow. The -reviewed workflows fall back to `GITHUB_TOKEN` when the App is not configured -(and then need that click); the tracker refuses to run without it. +tracker requires both App secrets before it runs. The reviewed sync workflows +continue to use `GITHUB_TOKEN` and require maintainer approval as before. To approve builds for an unvouched contributor's PR, apply **`build-approved`**. Until approval, the PR shows **Awaiting build approval** and its required diff --git a/bin/sync-upstream b/bin/sync-upstream index a08f184..5aae830 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -1093,11 +1093,12 @@ enforce_branch_lockstep() { groups["$key"]+="$package " done for key in "${!groups[@]}"; do - local members commits + local members commits pin read -r -a members <<<"${groups[$key]}" (( ${#members[@]} > 1 )) || continue commits=$(for package in "${members[@]}"; do - grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" + pin=$(grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" || true) + printf '%s\n' "${pin:-missing:$package}" done | sort -u | grep -c .) (( commits > 1 )) || continue print_error "Lockstep violation on $key: ${members[*]} pin different commits; restoring all of them" @@ -1115,17 +1116,34 @@ enforce_branch_lockstep() { sync_in_lane() { local package="$1" package_dir="$PKGBUILDS_DIR/$1" + snapshot_package "$package" if [[ -d "$package_dir" ]] && ! package_in_lane "$package_dir" "$LANE"; then print_info "Skipping $package: not in the $LANE lane" ((++SKIPPED)) return 0 fi - snapshot_package "$package" sync_package "$package" } if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then SPECIFIC_MODE=true + # A targeted run is still a branch update: include every sibling watching + # the same branch, otherwise the lockstep check never sees the omitted one. + declare -A selected=() selected_branches=() + for package in "${SPECIFIC_PACKAGES[@]}"; do + selected["$package"]=1 + key=$(branch_watch_key "$PKGBUILDS_DIR/$package" || true) + [[ -z "$key" ]] || selected_branches["$key"]=1 + done + for package_dir in "$PKGBUILDS_DIR"/*; do + [[ -f "$package_dir/PKGBUILD" ]] || continue + package=${package_dir##*/} + [[ -z "${selected[$package]:-}" ]] || continue + key=$(branch_watch_key "$package_dir" || true) + if [[ -n "$key" && -n "${selected_branches[$key]:-}" ]]; then + SPECIFIC_PACKAGES+=("$package") + fi + done for package in "${SPECIFIC_PACKAGES[@]}"; do sync_in_lane "$package" done diff --git a/docs/upstream-sources.md b/docs/upstream-sources.md index 79cff76..1749ee9 100644 --- a/docs/upstream-sources.md +++ b/docs/upstream-sources.md @@ -72,16 +72,40 @@ uses it. The Omarchy dev pair uses `{version}.r{count}.g{commit:.7}` instead because its published history counted every commit and the number must never go down. -`min_release_age` on a branch watch selects the newest commit that has been on -the branch for at least that long, so a burst of pushes builds once after it -settles rather than once per push. `BYPASS_MIN_RELEASE_AGE=1` takes the tip. +`min_release_age` holds a branch tip until its commit timestamp is old enough. +A fresh tip leaves the existing pin alone; the watch never walks backward to +an older commit. This uses Git's committer date, not the time a commit was +pushed. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold. Packages marked `"auto_merge": true` ride the unattended lane (`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened and auto-merged as soon as the build checks pass. `bin/sync-upstream --lane reviewed|auto-merge|all` selects a lane; the scheduled workflows each pass their own. Packages that pin the same branch move in lockstep: if one of them fails -to update, the run restores the others and reports the group as failed. +to update, the run restores the others and reports the group as failed. A +targeted sync includes the other packages watching that branch, so requesting +only `omarchy-dev` also updates `omarchy-settings-dev`. + +### Enable unattended branch updates + +The schedule already runs in GitHub Actions; no server cron job is needed. +It needs a GitHub App identity so its PRs trigger builds and its merges trigger +publishing without manual approval: + +1. [Create an organization GitHub App](https://github.com/organizations/omacom/settings/apps/new). + Use this repository's URL as the homepage, disable webhooks, and grant only + repository **Contents: Read and write** and **Pull requests: Read and write** + (Metadata read access is automatic). Limit installation to this organization. +2. Install the App on **omacom/omarchy-pkgs** only. +3. Generate a private key from the App's settings. In the repository's + [Actions secrets](https://github.com/omacom/omarchy-pkgs/settings/secrets/actions), + save the App ID as `PKGS_BOT_APP_ID` and the PEM key contents as + `PKGS_BOT_PRIVATE_KEY`. +4. Keep **Allow auto-merge** enabled and require `result`, `self-tests`, and + `build-isolation` on `master`; the App does not need a protection bypass. +5. After merging the tracker, run **Track upstream branches** once from Actions + to verify that its PR builds, auto-merges, and starts **Publish merged packages**. + Subsequent runs happen every two hours. Checksums retain their algorithms (SHA256, SHA512, BLAKE2, etc.) and source order. Changed git sources are hashed with makepkg's git-archive convention. Unchanged diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index 05828f4..f8468ab 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -184,10 +184,9 @@ def matches(watch, text, extra=None, full=False): yield candidate(watch, {**(extra or {}), **match.groupdict()}) -def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): - """Describe the newest commit on an upstream branch that has sat there for - at least min_age seconds (the branch analogue of "the newest release older - than the window ships"): commit, total count, date, and with a tag_pattern +def git_branch_tip(url, branch, tag_pattern, cache): + """Describe the current tip of an upstream branch: + commit, total count, date, and with a tag_pattern the newest release tag reachable from it plus the distance from that tag, so a branch build can be versioned .r.g, above the release it follows and below the next one, the way a pkgver() function would. @@ -195,7 +194,8 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): One blobless single-branch clone per (url, branch) per run, shared by every package that tracks it, so two recipes pinned from one clone always see the same commit. The clone is read with git only; nothing in it runs. - Returns None when every commit is younger than the window. + select_release applies the age hold to this tip, without walking back + into history (which could select a commit from a merged side branch). """ https(url) key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest() @@ -205,14 +205,7 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True) scratch.replace(work) git = ["git", "-C", str(work)] - selector = ["HEAD"] - if min_age: - cutoff = (now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(seconds=min_age) - selector = ["-1", f"--before={cutoff.isoformat()}", "HEAD"] - commit = run([*git, "rev-list", *selector], text=True).split()[:1] - if not commit: - return None - commit = commit[0] + commit = run([*git, "rev-parse", "HEAD"], text=True).strip() if not re.fullmatch(r"[0-9a-f]{40}", commit): raise ValueError("branch tip is not a commit") count = run([*git, "rev-list", "--count", commit], text=True).strip() @@ -238,7 +231,7 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None): return values -def discover(watch, fetch, min_age=0): +def discover(watch, fetch): provider = validate(watch) feed = watch[provider] results = [] @@ -265,9 +258,7 @@ def discover(watch, fetch, min_age=0): for tag, commit in tags.items(): results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True)) elif provider == "git_branch": - tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age) - if tip is None: - return [] # nothing has settled for min_age yet: wait, not an error + tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache) results.append(candidate(watch, tip)) elif provider == "npm": data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe="")) @@ -543,7 +534,7 @@ def sync(package, fetch, min_age=0, check=False): original = path.read_text() before = read_recipe(path) bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1" - release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass) + release = select_release(discover(watch, fetch), min_age, bypass=bypass) if release is None: return {"status": "skipped", "reason": "minimum release age"} current = scalar(before, "pkgver") diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index 4861e16..2921f7c 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -232,19 +232,94 @@ b2sums=('old' 'local-b2') with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'): w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P[0-9.]+)', self.root / 'other-cache') - def test_git_branch_min_age_selects_the_newest_settled_commit(self): + def test_git_branch_min_age_holds_the_tip_instead_of_selecting_history(self): repo, shas = self.branch_fixture(fresh_tip=True) - with self.redirect_clone(repo): - tip = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache) - settled = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P[0-9.]+)', self.fetch.cache, min_age=3600) - nothing = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache, min_age=10 ** 9) - self.assertEqual(tip['commit'], shas[-1], 'no window: the fresh tip') - self.assertEqual((settled['commit'], settled['distance'], settled['count']), (shas[-2], '2', '5'), 'one hour window: the commit before it') - self.assertIsNone(nothing, 'a window older than every commit selects nothing') watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'} with self.redirect_clone(repo): - self.assertEqual(w.discover(watch, self.fetch, min_age=10 ** 9), []) - self.assertEqual(w.discover(watch, self.fetch, min_age=3600)[0]['values']['commit'], shas[-2]) + releases = w.discover(watch, self.fetch) + self.assertEqual(w.select_release(releases)['values']['commit'], shas[-1]) + self.assertIsNone(w.select_release(releases, min_age=3600)) + self.assertEqual(w.select_release(releases, min_age=3600, bypass=True)['values']['commit'], shas[-1]) + + def branch_sync_fixture(self): + repo, shas = self.branch_fixture() + for directory in ['bin', 'helpers']: + shutil.copytree(ROOT / directory, self.root / directory) + for name in ['dev', 'settings-dev']: + package = self.root / 'pkgbuilds' / name + (package / '.omarchy').mkdir(parents=True) + (package / '.omarchy/package.json').write_text(json.dumps({ + 'source': 'local', 'auto_merge': True, 'upstream': {'watch': { + 'git_branch': 'https://example.test/tool.git', 'branch': 'main', + 'tag_pattern': r'v(?P[0-9.]+)', + 'version': '{version}.r{count}.g{commit:.7}', + 'variables': {'_commit': '{commit}'}}}})) + (package / 'PKGBUILD').write_text(f'''pkgname={name} +pkgver=1.0.0 +pkgrel=1 +_commit={shas[1]} +arch=('any') +source=("tool::git+https://example.test/tool.git#commit=${{_commit}}") +sha256sums=('old') +''') + stub = self.root / 'stub' + stub.mkdir() + git = stub / 'git' + git.write_text('''#!/usr/bin/env python3 +import os, sys +args = [os.environ['BRANCH_FIXTURE'] if arg == 'https://example.test/tool.git' else arg for arg in sys.argv[1:]] +os.execv(os.environ['REAL_GIT'], ['git', *args]) +''') + git.chmod(0o755) + env = {**os.environ, 'PATH': str(stub) + os.pathsep + os.environ['PATH'], + 'BRANCH_FIXTURE': f'file://{repo}', 'REAL_GIT': shutil.which('git')} + def sync(*args): + return subprocess.run([str(self.root / 'bin/sync-upstream'), *args], + env=env, text=True, capture_output=True) + return self.root / 'pkgbuilds', shas[-1], sync + + def test_targeted_branch_sync_updates_siblings_and_then_noops(self): + packages, tip, sync = self.branch_sync_fixture() + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + for package in packages.iterdir(): + recipe = w.read_recipe(package / 'PKGBUILD') + self.assertEqual(w.scalar(recipe, '_commit'), tip) + self.assertEqual(w.scalar(recipe, 'pkgver'), f'1.1.0.r5.g{tip[:7]}') + self.assertRegex(recipe['sha256sums'][0], r'^[0-9a-f]{64}$') + self.assertIn('Updated: 2', result.stdout) + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('Updated: 0', result.stdout) + + def test_branch_sync_rolls_back_when_a_sibling_fails(self): + packages, tip, sync = self.branch_sync_fixture() + broken = packages / 'settings-dev/PKGBUILD' + broken.write_text(broken.read_text().replace("sha256sums=('old')", 'sha256sums=()')) + before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')} + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn('Lockstep violation', result.stdout + result.stderr) + self.assertIn('Updated: 0', result.stdout) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + + def test_reviewed_lane_leaves_auto_merge_packages_untouched(self): + packages, tip, sync = self.branch_sync_fixture() + before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')} + result = sync('--lane', 'reviewed') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('Updated: 0', result.stdout) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + + def test_different_lanes_cannot_split_a_branch_pair(self): + packages, tip, sync = self.branch_sync_fixture() + metadata = packages / 'settings-dev/.omarchy/package.json' + metadata.write_text(metadata.read_text().replace('"auto_merge": true', '"auto_merge": false')) + before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')} + result = sync('--lane', 'auto-merge', 'dev') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn('Lockstep violation', result.stdout + result.stderr) + self.assertEqual(before, {p: p.read_bytes() for p in before}) def test_git_branch_tag_template_requires_tag_pattern(self): base = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'}