Add bin/sync-upstream for packages that track a vendor release feed
Some vendors publish a release feed of their own that is faster and more precise than anyone's packaging of it. A package opts in with an .omarchy/upstream.sh hook that reports the newest release as JSON, and the driver rewrites pkgver, the checksum arrays the hook names, and pkgrel. Writes are guarded on both ends: every assignment the update will touch is verified to exist before anything is written, so a hook naming an array the PKGBUILD lacks fails with the file untouched rather than half rewritten; and pkgver is held to pacman's character set, because it lands in a file makepkg sources as shell. Ordering is vercmp's, not sort -V's -- they disagree about whether 1.0a precedes 1.0, and pacman is what decides if a published package is an upgrade. That is also why the workflow runs in an Arch container rather than straight on the runner. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
David Heinemeier Hansson
co-authored by
Claude Opus 5
Claude Opus 5
parent
5575275941
commit
01a566f01a
@@ -0,0 +1,95 @@
|
||||
name: Sync Upstream Releases
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 6 hours, off the hour to dodge the scheduling backlog at :00
|
||||
- cron: '20 */6 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
packages:
|
||||
description: 'Specific packages to update (space-separated, leave empty for all)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Runs in an Arch container for vercmp: whether a release is an upgrade has
|
||||
# to be decided by the same comparator pacman will use on users' machines.
|
||||
- name: Update packages from upstream release feeds
|
||||
run: |
|
||||
docker run --rm \
|
||||
-e PACKAGES="$PACKAGES" \
|
||||
-e HOST_UID="$(id -u)" \
|
||||
-e HOST_GID="$(id -g)" \
|
||||
-v "$PWD/bin:/workspace/bin:ro" \
|
||||
-v "$PWD/helpers:/workspace/helpers:ro" \
|
||||
-v "$PWD/pkgbuilds:/workspace/pkgbuilds" \
|
||||
-w /workspace \
|
||||
archlinux:base-devel bash -lc '
|
||||
set -euo pipefail
|
||||
|
||||
pacman -Syu --noconfirm jq
|
||||
|
||||
groupadd -g "$HOST_GID" runner
|
||||
useradd -m -u "$HOST_UID" -g "$HOST_GID" runner
|
||||
chown -R runner:runner /workspace/pkgbuilds
|
||||
|
||||
if [[ -n "${PACKAGES:-}" ]]; then
|
||||
read -r -a package_args <<< "$PACKAGES"
|
||||
runuser -u runner -- ./bin/sync-upstream "${package_args[@]}"
|
||||
else
|
||||
runuser -u runner -- ./bin/sync-upstream
|
||||
fi
|
||||
'
|
||||
env:
|
||||
PACKAGES: ${{ github.event.inputs.packages }}
|
||||
|
||||
- name: Check for changes
|
||||
id: changes
|
||||
run: |
|
||||
if [ -z "$(git status --porcelain)" ]; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.changes.outputs.has_changes == 'true'
|
||||
uses: peter-evans/create-pull-request@v7
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
commit-message: 'chore: sync upstream releases'
|
||||
title: 'chore: sync upstream releases'
|
||||
body: |
|
||||
Automated update of packages that track an upstream vendor release
|
||||
feed rather than the AUR.
|
||||
|
||||
Each package reports its newest release through
|
||||
`.omarchy/upstream.sh`.
|
||||
branch: auto/sync-upstream
|
||||
delete-branch: true
|
||||
labels: automated
|
||||
reviewers: ryanrhughes
|
||||
|
||||
- name: Notify Basecamp on failure
|
||||
if: failure() && env.BASECAMP_CHATBOT_URL != ''
|
||||
env:
|
||||
BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }}
|
||||
run: |
|
||||
curl -s -o /dev/null \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$(jq -n --arg content \
|
||||
"🔴 <strong>Upstream sync failed</strong><br><a href=\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\">View run</a>" \
|
||||
'{content: $content}')" \
|
||||
"$BASECAMP_CHATBOT_URL"
|
||||
@@ -15,6 +15,7 @@ The filesystem no longer encodes release policy. Instead:
|
||||
- all other packages reach `stable` by promoting tested edge artifacts with `bin/repo migrate`
|
||||
- AUR sync behavior is controlled by `source`, `sync`, `aur`, patches, and hooks in `.omarchy/`
|
||||
- packages can opt out of unscoped builds with `skip_build`; explicit `--package` builds remain available
|
||||
- packages that follow a vendor release feed instead of the AUR carry an `.omarchy/upstream.sh` hook
|
||||
|
||||
## Prerequisites
|
||||
### aarch64 Builds (Optional)
|
||||
@@ -248,6 +249,40 @@ bin/sync-aur yay v4l2-relayd # Sync specific packages
|
||||
|
||||
AUR sync is metadata-driven. It preserves `.omarchy/`, replaces the package root with AUR contents, applies `.omarchy/patches/*.patch`, runs `.omarchy/post-sync.sh` when present, applies pkgrel metadata, removes AUR-only `.SRCINFO` and `.gitignore` files, and records `upstream_commit`.
|
||||
|
||||
### Sync Upstream Releases
|
||||
|
||||
```bash
|
||||
bin/sync-upstream # Update every package with an upstream hook
|
||||
bin/sync-upstream openai-codex-desktop # Update specific packages
|
||||
```
|
||||
|
||||
Some vendors publish a release feed of their own that is faster and more precise
|
||||
than the AUR packaging of it. Those packages are `source: local` — Omarchy owns
|
||||
the PKGBUILD — and provide `.omarchy/upstream.sh`, a hook that reports the newest
|
||||
upstream release as JSON on stdout:
|
||||
|
||||
```json
|
||||
{
|
||||
"pkgver": "1.2.3",
|
||||
"sha256sums": { "x86_64": ["<sha256>"], "aarch64": ["<sha256>"] }
|
||||
}
|
||||
```
|
||||
|
||||
Architecture keys become `sha256sums_<arch>` in the PKGBUILD; the key `any` means
|
||||
the unsuffixed `sha256sums` array, and only the arrays a hook names are touched.
|
||||
An empty object (`{}`) reports no update, which is how a hook waits out a release
|
||||
that has landed for one architecture but not yet the other.
|
||||
|
||||
When the reported version is newer than the checked-in one, `bin/sync-upstream`
|
||||
rewrites `pkgver` and those checksum arrays and resets `pkgrel` to 1. A version
|
||||
that is equal or older leaves the package alone, so a vendor rolling a release
|
||||
back cannot walk the repository backwards.
|
||||
|
||||
Hooks should read checksums from whatever manifest the vendor publishes rather
|
||||
than downloading the artifacts — see `pkgbuilds/openai-codex-desktop/.omarchy/upstream.sh`,
|
||||
which reads OpenAI's Debian package index and never fetches the 750 MB of debs
|
||||
it describes.
|
||||
|
||||
### Other
|
||||
|
||||
```bash
|
||||
@@ -260,6 +295,7 @@ bin/repo push # Upload local builds to the host and publi
|
||||
bin/add-package <package> # Add an AUR/local package with metadata
|
||||
bin/package-worktree <package> # Create upstream/patched/current scratch workspace
|
||||
bin/repo remove <package> # Remove package
|
||||
bin/sync-upstream # Update packages that track a vendor release feed
|
||||
bin/clean-docker # Clear Docker images/cache (forces fresh rebuild)
|
||||
```
|
||||
|
||||
@@ -345,7 +381,8 @@ omarchy-pkgs/
|
||||
│ └── .omarchy/
|
||||
│ ├── package.json # Source/sync/release metadata
|
||||
│ ├── patches/ # Omarchy patches reapplied after AUR sync
|
||||
│ └── post-sync.sh # Optional dynamic post-sync customization hook
|
||||
│ ├── post-sync.sh # Optional dynamic post-sync customization hook
|
||||
│ └── upstream.sh # Optional vendor release feed hook (non-AUR packages)
|
||||
├── build/
|
||||
├── build-output/ # Unsigned packages (temporary)
|
||||
│ ├── edge/
|
||||
@@ -396,7 +433,7 @@ Minimal examples:
|
||||
|
||||
Fields:
|
||||
|
||||
- `source`: `aur` or `local`
|
||||
- `source`: `aur` or `local`. A `local` package can still follow an upstream release with an `.omarchy/upstream.sh` hook.
|
||||
- `sync`: optional for AUR packages; defaults to `true`. Set `false` for AUR-origin packages that Omarchy maintains manually.
|
||||
- `aur`: optional AUR package name when it differs from the local package directory, usually for split packages.
|
||||
- `release_ring`: optional. `fast` means the package is built directly for stable as well as edge. Packages without a ring build in edge and reach stable through tested artifact promotion (`bin/repo migrate`).
|
||||
@@ -539,6 +576,7 @@ The repository includes GitHub workflows and systemd services for automated rele
|
||||
#### GitHub Workflows
|
||||
|
||||
1. **sync-aur.yml** (Every 6 hours): Syncs AUR packages according to `.omarchy/package.json` and opens a PR when changes are found.
|
||||
2. **sync-upstream.yml** (Every 6 hours): Runs `.omarchy/upstream.sh` for packages that track a vendor release feed and opens a PR when a newer version is out.
|
||||
|
||||
#### Systemd Services
|
||||
|
||||
|
||||
Executable
+314
@@ -0,0 +1,314 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
||||
source "$BUILD_ROOT/helpers/message-helpers.sh"
|
||||
source "$BUILD_ROOT/helpers/paths.sh"
|
||||
source "$BUILD_ROOT/helpers/package-metadata.sh"
|
||||
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TEMP_DIR"' EXIT
|
||||
|
||||
SPECIFIC_PACKAGES=()
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [PACKAGE...]
|
||||
|
||||
Update packages that track an upstream vendor release feed instead of the AUR.
|
||||
|
||||
A package opts in by providing pkgbuilds/<package>/.omarchy/upstream.sh, a hook
|
||||
that reports the newest upstream release as JSON on stdout:
|
||||
|
||||
{
|
||||
"pkgver": "1.2.3",
|
||||
"sha256sums": { "x86_64": ["<sha256>"], "aarch64": ["<sha256>"] }
|
||||
}
|
||||
|
||||
Architecture keys become sha256sums_<arch> in the PKGBUILD; the key "any" means
|
||||
the unsuffixed sha256sums array. An empty object ({}) reports no update.
|
||||
|
||||
When the reported version is newer than the checked-in one, pkgver and the
|
||||
listed checksum arrays are rewritten and pkgrel is reset to 1.
|
||||
|
||||
Arguments:
|
||||
PACKAGE One or more package names to update (optional)
|
||||
|
||||
Examples:
|
||||
$0 # Update every package with an upstream hook
|
||||
$0 openai-codex-desktop # Update specific packages
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--*)
|
||||
print_error "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
SPECIFIC_PACKAGES+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! command -v vercmp >/dev/null 2>&1; then
|
||||
print_error "vercmp not found: this needs pacman to decide whether a release is an upgrade"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
print_header "Upstream Package Sync"
|
||||
|
||||
UPDATED=0
|
||||
SKIPPED=0
|
||||
FAILED=0
|
||||
SPECIFIC_MODE=false
|
||||
|
||||
get_pkgver() {
|
||||
local package_dir="$1"
|
||||
|
||||
grep -m1 '^pkgver=' "$package_dir/PKGBUILD" | cut -d= -f2- | tr -d "\"'"
|
||||
}
|
||||
|
||||
assert_single_assignment() {
|
||||
local pkgbuild="$1"
|
||||
local pattern="$2"
|
||||
local label="$3"
|
||||
|
||||
if [[ $(grep -c "$pattern" "$pkgbuild") -ne 1 ]]; then
|
||||
print_error "Expected exactly one $label assignment in $pkgbuild"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
set_pkgbuild_scalar() {
|
||||
local pkgbuild="$1"
|
||||
local field="$2"
|
||||
local value="$3"
|
||||
|
||||
assert_single_assignment "$pkgbuild" "^${field}=" "$field" || return 1
|
||||
sed -i "s/^${field}=.*/${field}=${value}/" "$pkgbuild"
|
||||
}
|
||||
|
||||
# Replace an array assignment, however many lines the original spans.
|
||||
set_pkgbuild_array() {
|
||||
local pkgbuild="$1"
|
||||
local name="$2"
|
||||
shift 2
|
||||
local values=("$@")
|
||||
|
||||
assert_single_assignment "$pkgbuild" "^${name}=(" "$name" || return 1
|
||||
|
||||
local block="$TEMP_DIR/array-block"
|
||||
if [[ ${#values[@]} -eq 1 ]]; then
|
||||
printf "%s=('%s')\n" "$name" "${values[0]}" > "$block"
|
||||
else
|
||||
printf '%s=(\n' "$name" > "$block"
|
||||
printf " '%s'\n" "${values[@]}" >> "$block"
|
||||
printf ')\n' >> "$block"
|
||||
fi
|
||||
|
||||
# Rewrite beside the PKGBUILD so the move is an atomic same-filesystem rename.
|
||||
local rewritten="$pkgbuild.sync-upstream"
|
||||
if ! awk -v prefix="${name}=(" -v block="$block" '
|
||||
!replaced && index($0, prefix) == 1 {
|
||||
while ((getline line < block) > 0) print line
|
||||
close(block)
|
||||
replaced = 1
|
||||
# A ")" anywhere past the opening closes the array; testing for one at end
|
||||
# of line instead would treat a trailing comment as a continuation and eat
|
||||
# every line up to the next ")".
|
||||
if (index(substr($0, length(prefix) + 1), ")") == 0) skipping = 1
|
||||
next
|
||||
}
|
||||
skipping { if ($0 ~ /\)/) skipping = 0; next }
|
||||
{ print }
|
||||
' "$pkgbuild" > "$rewritten"; then
|
||||
print_error "Failed to rewrite ${name} in $pkgbuild"
|
||||
rm -f "$rewritten"
|
||||
return 1
|
||||
fi
|
||||
|
||||
chmod --reference="$pkgbuild" "$rewritten"
|
||||
mv "$rewritten" "$pkgbuild"
|
||||
}
|
||||
|
||||
# pacman's own comparator, because nothing else agrees with it at the corners:
|
||||
# sort -V calls 1.0a newer than 1.0, vercmp calls it older, and pacman is what
|
||||
# decides whether a published package is an upgrade.
|
||||
version_is_newer() {
|
||||
local candidate="$1"
|
||||
local current="$2"
|
||||
|
||||
[[ "$candidate" != "$current" ]] || return 1
|
||||
[[ "$(vercmp "$candidate" "$current")" -gt 0 ]]
|
||||
}
|
||||
|
||||
validate_release() {
|
||||
local release="$1"
|
||||
|
||||
# pkgver is written into the PKGBUILD, which makepkg sources as shell, so it
|
||||
# is held to pacman's own character set rather than merely being non-empty.
|
||||
jq -e '
|
||||
(.pkgver | type == "string" and test("^[A-Za-z0-9._+]+$"))
|
||||
and (.sha256sums | type == "object" and length > 0)
|
||||
and (.sha256sums | to_entries | all(
|
||||
.key | test("^[a-z0-9_]+$")
|
||||
))
|
||||
and (.sha256sums | to_entries | all(
|
||||
.value | type == "array" and length > 0 and all(test("^[0-9a-f]{64}$"))
|
||||
))
|
||||
' <<<"$release" >/dev/null
|
||||
}
|
||||
|
||||
apply_release() {
|
||||
local package_dir="$1"
|
||||
local release="$2"
|
||||
local pkgver="$3"
|
||||
local pkgbuild="$package_dir/PKGBUILD"
|
||||
|
||||
local arch array values
|
||||
local targets=()
|
||||
|
||||
while IFS= read -r arch; do
|
||||
if [[ "$arch" == "any" ]]; then
|
||||
array="sha256sums"
|
||||
else
|
||||
array="sha256sums_$arch"
|
||||
fi
|
||||
targets+=("$arch:$array")
|
||||
done < <(jq -r '.sha256sums | keys[]' <<<"$release")
|
||||
|
||||
# Everything the update will touch is checked before anything is written. A
|
||||
# hook naming an array the PKGBUILD does not have must fail with the file
|
||||
# untouched rather than half rewritten.
|
||||
assert_single_assignment "$pkgbuild" '^pkgver=' pkgver || return 1
|
||||
assert_single_assignment "$pkgbuild" '^pkgrel=' pkgrel || return 1
|
||||
local target
|
||||
for target in "${targets[@]}"; do
|
||||
assert_single_assignment "$pkgbuild" "^${target#*:}=(" "${target#*:}" || return 1
|
||||
done
|
||||
|
||||
for target in "${targets[@]}"; do
|
||||
arch="${target%%:*}"
|
||||
array="${target#*:}"
|
||||
|
||||
mapfile -t values < <(jq -r --arg arch "$arch" '.sha256sums[$arch][]' <<<"$release")
|
||||
set_pkgbuild_array "$pkgbuild" "$array" "${values[@]}" || return 1
|
||||
done
|
||||
|
||||
set_pkgbuild_scalar "$pkgbuild" pkgver "$pkgver" || return 1
|
||||
set_pkgbuild_scalar "$pkgbuild" pkgrel 1 || return 1
|
||||
}
|
||||
|
||||
sync_package() {
|
||||
local package="$1"
|
||||
local package_dir="$PKGBUILDS_DIR/$package"
|
||||
local hook="$package_dir/.omarchy/upstream.sh"
|
||||
|
||||
if [[ ! -f "$package_dir/PKGBUILD" ]]; then
|
||||
print_error "Package $package has no PKGBUILD"
|
||||
((++FAILED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ ! -f "$hook" ]]; then
|
||||
if [[ "$SPECIFIC_MODE" == true ]]; then
|
||||
print_error "Package $package is missing .omarchy/upstream.sh"
|
||||
((++FAILED))
|
||||
else
|
||||
print_info "Skipping $package: no upstream hook"
|
||||
((++SKIPPED))
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
print_info "Checking $package for upstream releases..."
|
||||
|
||||
local release
|
||||
if ! release=$(cd "$package_dir" && PACKAGE_NAME="$package" bash .omarchy/upstream.sh); then
|
||||
print_error "Upstream hook failed for $package"
|
||||
((++FAILED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! jq -e . >/dev/null 2>&1 <<<"$release"; then
|
||||
print_error "Upstream hook for $package did not report valid JSON"
|
||||
((++FAILED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$(jq -r 'has("pkgver")' <<<"$release")" == "false" ]]; then
|
||||
print_info " No upstream update reported"
|
||||
((++SKIPPED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! validate_release "$release"; then
|
||||
print_error "Upstream hook for $package reported a malformed release"
|
||||
((++FAILED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pkgver current_pkgver
|
||||
pkgver=$(jq -r '.pkgver' <<<"$release")
|
||||
current_pkgver=$(get_pkgver "$package_dir")
|
||||
|
||||
if [[ -z "$current_pkgver" ]]; then
|
||||
print_error "Could not read pkgver from $package_dir/PKGBUILD"
|
||||
((++FAILED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$pkgver" == "$current_pkgver" ]]; then
|
||||
print_info " Already at $current_pkgver"
|
||||
((++SKIPPED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! version_is_newer "$pkgver" "$current_pkgver"; then
|
||||
print_warning " Upstream reports $pkgver, older than the checked-in $current_pkgver; leaving it alone"
|
||||
((++SKIPPED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! apply_release "$package_dir" "$release" "$pkgver"; then
|
||||
print_error "Failed to update $package"
|
||||
((++FAILED))
|
||||
return 0
|
||||
fi
|
||||
|
||||
print_success " $current_pkgver -> $pkgver"
|
||||
((++UPDATED))
|
||||
}
|
||||
|
||||
if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then
|
||||
SPECIFIC_MODE=true
|
||||
for package in "${SPECIFIC_PACKAGES[@]}"; do
|
||||
sync_package "$package"
|
||||
done
|
||||
else
|
||||
while IFS= read -r package; do
|
||||
sync_package "$package"
|
||||
done < <(packages_for_upstream_sync)
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ $FAILED -gt 0 ]]; then
|
||||
print_error "Upstream sync completed with failures"
|
||||
else
|
||||
print_success "Upstream sync complete!"
|
||||
fi
|
||||
echo " Target: $PKGBUILDS_DIR"
|
||||
echo " Updated: $UPDATED"
|
||||
echo " Skipped: $SKIPPED"
|
||||
echo " Failed: $FAILED"
|
||||
|
||||
if [[ $FAILED -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
@@ -135,6 +135,19 @@ packages_for_aur_sync() {
|
||||
done
|
||||
}
|
||||
|
||||
package_has_upstream_hook() {
|
||||
local pkgdir="$1"
|
||||
[[ -f "$pkgdir/.omarchy/upstream.sh" ]]
|
||||
}
|
||||
|
||||
packages_for_upstream_sync() {
|
||||
package_dirs | while IFS= read -r pkgdir; do
|
||||
if package_has_upstream_hook "$pkgdir"; then
|
||||
basename "$pkgdir"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
packages_for_mirror() {
|
||||
local mirror="$1"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user