Keep branch syncs together and simplify tracker setup

This commit is contained in:
Ryan Hughes committed 2026-09-27 12:39:53 -04:00
1 parent d87686ca4f
commit 7fe542cde7
8 files changed
+165 -98

No files matched your search

+2 -18
View File
@@ -68,27 +68,11 @@ jobs:
echo "has_changes=true" >> "$GITHUB_OUTPUT"
fi
# App token rather than GITHUB_TOKEN so the PR's build and test runs
# start without a maintainer approving them (see sync-upstream.yml).
- name: Mint the bot token
if: steps.changes.outputs.has_changes == 'true'
id: app
env:
PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }}
# Without the App configured this falls back to GITHUB_TOKEN below,
# which still opens the PR; a maintainer then has to approve its
# workflow runs by hand, as before.
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
app-id: ${{ secrets.PKGS_BOT_APP_ID }}
private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }}
continue-on-error: true
- name: Create Pull Request
if: steps.changes.outputs.has_changes == 'true'
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
uses: peter-evans/create-pull-request@v7
with:
token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }}
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: 'chore: rebuild against updated dependencies'
title: 'chore: rebuild against updated dependencies'
body: |
+2 -21
View File
@@ -72,30 +72,11 @@ jobs:
echo "has_changes=true" >> "$GITHUB_OUTPUT"
fi
# A PR opened with GITHUB_TOKEN gets its build and test runs held
# until a maintainer clicks "Approve workflows to run"; one opened by
# the App builds on its own, so the reviewer sees a green (or red) PR
# instead of a pending one. The App only opens the PR: merging stays
# a human decision in this lane.
- name: Mint the bot token
if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
id: app
env:
PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }}
# Without the App configured this falls back to GITHUB_TOKEN below,
# which still opens the PR; a maintainer then has to approve its
# workflow runs by hand, as before.
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
app-id: ${{ secrets.PKGS_BOT_APP_ID }}
private-key: ${{ secrets.PKGS_BOT_PRIVATE_KEY }}
continue-on-error: true
- name: Create Pull Request
if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
uses: peter-evans/create-pull-request@v7
with:
token: ${{ steps.app.outputs.token || secrets.GITHUB_TOKEN }}
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: 'chore: sync upstream releases'
title: 'chore: sync upstream releases'
body: |
+14 -20
View File
@@ -39,6 +39,16 @@ jobs:
contents: read
steps:
- name: Require the bot App
env:
PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }}
PKGS_BOT_PRIVATE_KEY: ${{ secrets.PKGS_BOT_PRIVATE_KEY }}
run: |
if [[ -z "$PKGS_BOT_APP_ID" || -z "$PKGS_BOT_PRIVATE_KEY" ]]; then
echo "::error::Set PKGS_BOT_APP_ID and PKGS_BOT_PRIVATE_KEY for a GitHub App installed on this repository with Contents: write and Pull requests: write."
exit 1
fi
- name: Checkout repository
uses: actions/checkout@v4
with:
@@ -93,19 +103,6 @@ jobs:
} >> "$GITHUB_STEP_SUMMARY"
fi
# No fallback to GITHUB_TOKEN here: a PR it opened would sit with its
# checks held, and an auto-merge it enabled would land without running
# publish.yml. Better to fail loudly than to pin quietly.
- name: Require the bot App
if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
env:
PKGS_BOT_APP_ID: ${{ secrets.PKGS_BOT_APP_ID }}
run: |
if [[ -z "$PKGS_BOT_APP_ID" ]]; then
echo "::error::PKGS_BOT_APP_ID / PKGS_BOT_PRIVATE_KEY are not set. Create a GitHub App with Contents: write and Pull requests: write, install it on this repository, and store its id and private key as those secrets."
exit 1
fi
- name: Mint the bot token
if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
id: app
@@ -126,7 +123,7 @@ jobs:
echo "title=Track upstream branches: ${title}" >> "$GITHUB_OUTPUT"
- name: Open or update the tracking PR
if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' }}
if: ${{ !cancelled() && steps.changes.outputs.has_changes == 'true' && steps.app.outcome == 'success' }}
id: pr
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
with:
@@ -136,8 +133,8 @@ jobs:
body: |
Automated pin of packages that follow a moving upstream branch
(`"auto_merge": true` in `.omarchy/package.json`). Each package's
`_commit` now points at the branch tip that has been there for at
least its `min_release_age`.
`_commit` now points at the branch tip. Fresh tips wait until
their commit timestamp is at least `min_release_age` old.
This PR auto-merges once the build checks pass. A failing build
leaves it open; the next tracker run replaces it with the newer tip.
@@ -159,10 +156,7 @@ jobs:
echo "auto-merge already enabled on #$PR"
exit 0
fi
# A PR whose checks all reused existing artifacts can be clean
# before this step runs; GitHub then refuses --auto, so merge it.
gh pr merge --auto --merge "$PR" -R "${{ github.repository }}" \
|| gh pr merge --merge "$PR" -R "${{ github.repository }}"
gh pr merge --auto --merge "$PR" -R "${{ github.repository }}"
- name: Notify Basecamp on failure
if: failure() && env.BASECAMP_CHATBOT_URL != ''
+4 -4
View File
@@ -891,15 +891,15 @@ The repository includes GitHub workflows and systemd services for automated rele
1. **sync-upstream.yml** (Every 6 hours): Watches direct upstream feeds and updates owned recipes on the reviewed lane. Successful package updates reach a PR even if another feed fails; failed recipes stay untouched and the workflow remains red.
2. **sync-rebuilds.yml** (Every 6 hours): Bumps pkgrel for packages whose `rebuild_on` dependencies have moved in the official repositories and opens a PR.
3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the newest tip of its watched branch that has sat there for `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it.
3. **track-branches.yml** (Every 2 hours): The unattended lane. Pins every `"auto_merge": true` package to the tip of its watched branch once its commit timestamp clears `min_release_age`, opens one PR for all of them, and enables auto-merge. Packages pinned from the same branch move together or not at all, including targeted syncs. The PR builds like any other; a tip that fails to build stays an open red PR until the next tick supersedes it.
The sync PRs are opened with a GitHub App token (`PKGS_BOT_APP_ID` and
The tracking PR is opened with a GitHub App token (`PKGS_BOT_APP_ID` and
`PKGS_BOT_PRIVATE_KEY` secrets; the App needs Contents and Pull requests
write on this repository). A PR opened with the workflow's own `GITHUB_TOKEN`
has its build and test runs held until a maintainer approves them, and an
auto-merge it enabled would land without running the publish workflow. The
reviewed workflows fall back to `GITHUB_TOKEN` when the App is not configured
(and then need that click); the tracker refuses to run without it.
tracker requires both App secrets before it runs. The reviewed sync workflows
continue to use `GITHUB_TOKEN` and require maintainer approval as before.
To approve builds for an unvouched contributor's PR, apply **`build-approved`**.
Until approval, the PR shows **Awaiting build approval** and its required
+21 -3
View File
@@ -1093,11 +1093,12 @@ enforce_branch_lockstep() {
groups["$key"]+="$package "
done
for key in "${!groups[@]}"; do
local members commits
local members commits pin
read -r -a members <<<"${groups[$key]}"
(( ${#members[@]} > 1 )) || continue
commits=$(for package in "${members[@]}"; do
grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'"
pin=$(grep -m1 -E '^_commit=' "$PKGBUILDS_DIR/$package/PKGBUILD" | cut -d= -f2- | tr -d "\"'" || true)
printf '%s\n' "${pin:-missing:$package}"
done | sort -u | grep -c .)
(( commits > 1 )) || continue
print_error "Lockstep violation on $key: ${members[*]} pin different commits; restoring all of them"
@@ -1115,17 +1116,34 @@ enforce_branch_lockstep() {
sync_in_lane() {
local package="$1" package_dir="$PKGBUILDS_DIR/$1"
snapshot_package "$package"
if [[ -d "$package_dir" ]] && ! package_in_lane "$package_dir" "$LANE"; then
print_info "Skipping $package: not in the $LANE lane"
((++SKIPPED))
return 0
fi
snapshot_package "$package"
sync_package "$package"
}
if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then
SPECIFIC_MODE=true
# A targeted run is still a branch update: include every sibling watching
# the same branch, otherwise the lockstep check never sees the omitted one.
declare -A selected=() selected_branches=()
for package in "${SPECIFIC_PACKAGES[@]}"; do
selected["$package"]=1
key=$(branch_watch_key "$PKGBUILDS_DIR/$package" || true)
[[ -z "$key" ]] || selected_branches["$key"]=1
done
for package_dir in "$PKGBUILDS_DIR"/*; do
[[ -f "$package_dir/PKGBUILD" ]] || continue
package=${package_dir##*/}
[[ -z "${selected[$package]:-}" ]] || continue
key=$(branch_watch_key "$package_dir" || true)
if [[ -n "$key" && -n "${selected_branches[$key]:-}" ]]; then
SPECIFIC_PACKAGES+=("$package")
fi
done
for package in "${SPECIFIC_PACKAGES[@]}"; do
sync_in_lane "$package"
done
+28 -4
View File
@@ -72,16 +72,40 @@ uses it. The Omarchy dev pair uses `{version}.r{count}.g{commit:.7}` instead
because its published history counted every commit and the number must never
go down.
`min_release_age` on a branch watch selects the newest commit that has been on
the branch for at least that long, so a burst of pushes builds once after it
settles rather than once per push. `BYPASS_MIN_RELEASE_AGE=1` takes the tip.
`min_release_age` holds a branch tip until its commit timestamp is old enough.
A fresh tip leaves the existing pin alone; the watch never walks backward to
an older commit. This uses Git's committer date, not the time a commit was
pushed. `BYPASS_MIN_RELEASE_AGE=1` bypasses the hold.
Packages marked `"auto_merge": true` ride the unattended lane
(`track-branches.yml`) instead of the reviewed sync PR: their bump PR is opened
and auto-merged as soon as the build checks pass. `bin/sync-upstream --lane
reviewed|auto-merge|all` selects a lane; the scheduled workflows each pass their
own. Packages that pin the same branch move in lockstep: if one of them fails
to update, the run restores the others and reports the group as failed.
to update, the run restores the others and reports the group as failed. A
targeted sync includes the other packages watching that branch, so requesting
only `omarchy-dev` also updates `omarchy-settings-dev`.
### Enable unattended branch updates
The schedule already runs in GitHub Actions; no server cron job is needed.
It needs a GitHub App identity so its PRs trigger builds and its merges trigger
publishing without manual approval:
1. [Create an organization GitHub App](https://github.com/organizations/omacom/settings/apps/new).
Use this repository's URL as the homepage, disable webhooks, and grant only
repository **Contents: Read and write** and **Pull requests: Read and write**
(Metadata read access is automatic). Limit installation to this organization.
2. Install the App on **omacom/omarchy-pkgs** only.
3. Generate a private key from the App's settings. In the repository's
[Actions secrets](https://github.com/omacom/omarchy-pkgs/settings/secrets/actions),
save the App ID as `PKGS_BOT_APP_ID` and the PEM key contents as
`PKGS_BOT_PRIVATE_KEY`.
4. Keep **Allow auto-merge** enabled and require `result`, `self-tests`, and
`build-isolation` on `master`; the App does not need a protection bypass.
5. After merging the tracker, run **Track upstream branches** once from Actions
to verify that its PR builds, auto-merges, and starts **Publish merged packages**.
Subsequent runs happen every two hours.
Checksums retain their algorithms (SHA256, SHA512, BLAKE2, etc.) and source order.
Changed git sources are hashed with makepkg's git-archive convention. Unchanged
+9 -18
View File
@@ -184,10 +184,9 @@ def matches(watch, text, extra=None, full=False):
yield candidate(watch, {**(extra or {}), **match.groupdict()})
def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None):
"""Describe the newest commit on an upstream branch that has sat there for
at least min_age seconds (the branch analogue of "the newest release older
than the window ships"): commit, total count, date, and with a tag_pattern
def git_branch_tip(url, branch, tag_pattern, cache):
"""Describe the current tip of an upstream branch:
commit, total count, date, and with a tag_pattern
the newest release tag reachable from it plus the distance from that tag,
so a branch build can be versioned <tag>.r<n>.g<sha>, above the release it
follows and below the next one, the way a pkgver() function would.
@@ -195,7 +194,8 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None):
One blobless single-branch clone per (url, branch) per run, shared by
every package that tracks it, so two recipes pinned from one clone always
see the same commit. The clone is read with git only; nothing in it runs.
Returns None when every commit is younger than the window.
select_release applies the age hold to this tip, without walking back
into history (which could select a commit from a merged side branch).
"""
https(url)
key = hashlib.sha256(f"{url}#{branch}".encode()).hexdigest()
@@ -205,14 +205,7 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None):
subprocess.run(["git", "clone", "--quiet", "--bare", "--filter=blob:none", "--single-branch", "--branch", branch, url, str(scratch)], check=True)
scratch.replace(work)
git = ["git", "-C", str(work)]
selector = ["HEAD"]
if min_age:
cutoff = (now or dt.datetime.now(dt.timezone.utc)) - dt.timedelta(seconds=min_age)
selector = ["-1", f"--before={cutoff.isoformat()}", "HEAD"]
commit = run([*git, "rev-list", *selector], text=True).split()[:1]
if not commit:
return None
commit = commit[0]
commit = run([*git, "rev-parse", "HEAD"], text=True).strip()
if not re.fullmatch(r"[0-9a-f]{40}", commit):
raise ValueError("branch tip is not a commit")
count = run([*git, "rev-list", "--count", commit], text=True).strip()
@@ -238,7 +231,7 @@ def git_branch_tip(url, branch, tag_pattern, cache, min_age=0, now=None):
return values
def discover(watch, fetch, min_age=0):
def discover(watch, fetch):
provider = validate(watch)
feed = watch[provider]
results = []
@@ -265,9 +258,7 @@ def discover(watch, fetch, min_age=0):
for tag, commit in tags.items():
results.extend(matches(watch, tag, {"tag": tag, "commit": commit}, full=True))
elif provider == "git_branch":
tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache, min_age)
if tip is None:
return [] # nothing has settled for min_age yet: wait, not an error
tip = git_branch_tip(feed, watch["branch"], watch.get("tag_pattern"), fetch.cache)
results.append(candidate(watch, tip))
elif provider == "npm":
data = fetch.json("https://registry.npmjs.org/" + quote(feed, safe=""))
@@ -543,7 +534,7 @@ def sync(package, fetch, min_age=0, check=False):
original = path.read_text()
before = read_recipe(path)
bypass = os.environ.get("BYPASS_MIN_RELEASE_AGE") == "1"
release = select_release(discover(watch, fetch, 0 if bypass else min_age), min_age, bypass=bypass)
release = select_release(discover(watch, fetch), min_age, bypass=bypass)
if release is None:
return {"status": "skipped", "reason": "minimum release age"}
current = scalar(before, "pkgver")
+85 -10
View File
@@ -232,19 +232,94 @@ b2sums=('old' 'local-b2')
with self.redirect_clone(repo), self.assertRaisesRegex(ValueError, 'no tag'):
w.git_branch_tip('https://example.test/tool.git', 'main', r'release-(?P<version>[0-9.]+)', self.root / 'other-cache')
def test_git_branch_min_age_selects_the_newest_settled_commit(self):
def test_git_branch_min_age_holds_the_tip_instead_of_selecting_history(self):
repo, shas = self.branch_fixture(fresh_tip=True)
with self.redirect_clone(repo):
tip = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache)
settled = w.git_branch_tip('https://example.test/tool.git', 'main', r'v(?P<version>[0-9.]+)', self.fetch.cache, min_age=3600)
nothing = w.git_branch_tip('https://example.test/tool.git', 'main', None, self.fetch.cache, min_age=10 ** 9)
self.assertEqual(tip['commit'], shas[-1], 'no window: the fresh tip')
self.assertEqual((settled['commit'], settled['distance'], settled['count']), (shas[-2], '2', '5'), 'one hour window: the commit before it')
self.assertIsNone(nothing, 'a window older than every commit selects nothing')
watch = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'}
with self.redirect_clone(repo):
self.assertEqual(w.discover(watch, self.fetch, min_age=10 ** 9), [])
self.assertEqual(w.discover(watch, self.fetch, min_age=3600)[0]['values']['commit'], shas[-2])
releases = w.discover(watch, self.fetch)
self.assertEqual(w.select_release(releases)['values']['commit'], shas[-1])
self.assertIsNone(w.select_release(releases, min_age=3600))
self.assertEqual(w.select_release(releases, min_age=3600, bypass=True)['values']['commit'], shas[-1])
def branch_sync_fixture(self):
repo, shas = self.branch_fixture()
for directory in ['bin', 'helpers']:
shutil.copytree(ROOT / directory, self.root / directory)
for name in ['dev', 'settings-dev']:
package = self.root / 'pkgbuilds' / name
(package / '.omarchy').mkdir(parents=True)
(package / '.omarchy/package.json').write_text(json.dumps({
'source': 'local', 'auto_merge': True, 'upstream': {'watch': {
'git_branch': 'https://example.test/tool.git', 'branch': 'main',
'tag_pattern': r'v(?P<version>[0-9.]+)',
'version': '{version}.r{count}.g{commit:.7}',
'variables': {'_commit': '{commit}'}}}}))
(package / 'PKGBUILD').write_text(f'''pkgname={name}
pkgver=1.0.0
pkgrel=1
_commit={shas[1]}
arch=('any')
source=("tool::git+https://example.test/tool.git#commit=${{_commit}}")
sha256sums=('old')
''')
stub = self.root / 'stub'
stub.mkdir()
git = stub / 'git'
git.write_text('''#!/usr/bin/env python3
import os, sys
args = [os.environ['BRANCH_FIXTURE'] if arg == 'https://example.test/tool.git' else arg for arg in sys.argv[1:]]
os.execv(os.environ['REAL_GIT'], ['git', *args])
''')
git.chmod(0o755)
env = {**os.environ, 'PATH': str(stub) + os.pathsep + os.environ['PATH'],
'BRANCH_FIXTURE': f'file://{repo}', 'REAL_GIT': shutil.which('git')}
def sync(*args):
return subprocess.run([str(self.root / 'bin/sync-upstream'), *args],
env=env, text=True, capture_output=True)
return self.root / 'pkgbuilds', shas[-1], sync
def test_targeted_branch_sync_updates_siblings_and_then_noops(self):
packages, tip, sync = self.branch_sync_fixture()
result = sync('--lane', 'auto-merge', 'dev')
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
for package in packages.iterdir():
recipe = w.read_recipe(package / 'PKGBUILD')
self.assertEqual(w.scalar(recipe, '_commit'), tip)
self.assertEqual(w.scalar(recipe, 'pkgver'), f'1.1.0.r5.g{tip[:7]}')
self.assertRegex(recipe['sha256sums'][0], r'^[0-9a-f]{64}$')
self.assertIn('Updated: 2', result.stdout)
result = sync('--lane', 'auto-merge', 'dev')
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn('Updated: 0', result.stdout)
def test_branch_sync_rolls_back_when_a_sibling_fails(self):
packages, tip, sync = self.branch_sync_fixture()
broken = packages / 'settings-dev/PKGBUILD'
broken.write_text(broken.read_text().replace("sha256sums=('old')", 'sha256sums=()'))
before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')}
result = sync('--lane', 'auto-merge', 'dev')
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertIn('Lockstep violation', result.stdout + result.stderr)
self.assertIn('Updated: 0', result.stdout)
self.assertEqual(before, {p: p.read_bytes() for p in before})
def test_reviewed_lane_leaves_auto_merge_packages_untouched(self):
packages, tip, sync = self.branch_sync_fixture()
before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')}
result = sync('--lane', 'reviewed')
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn('Updated: 0', result.stdout)
self.assertEqual(before, {p: p.read_bytes() for p in before})
def test_different_lanes_cannot_split_a_branch_pair(self):
packages, tip, sync = self.branch_sync_fixture()
metadata = packages / 'settings-dev/.omarchy/package.json'
metadata.write_text(metadata.read_text().replace('"auto_merge": true', '"auto_merge": false'))
before = {p: p.read_bytes() for p in packages.glob('*/PKGBUILD')}
result = sync('--lane', 'auto-merge', 'dev')
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertIn('Lockstep violation', result.stdout + result.stderr)
self.assertEqual(before, {p: p.read_bytes() for p in before})
def test_git_branch_tag_template_requires_tag_pattern(self):
base = {'git_branch': 'https://example.test/tool.git', 'branch': 'main'}