From 01a566f01adba272a151b53bb3e50e053e6ca91d Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Sat, 15 Aug 2026 08:18:25 -0700 Subject: [PATCH] Add bin/sync-upstream for packages that track a vendor release feed Some vendors publish a release feed of their own that is faster and more precise than anyone's packaging of it. A package opts in with an .omarchy/upstream.sh hook that reports the newest release as JSON, and the driver rewrites pkgver, the checksum arrays the hook names, and pkgrel. Writes are guarded on both ends: every assignment the update will touch is verified to exist before anything is written, so a hook naming an array the PKGBUILD lacks fails with the file untouched rather than half rewritten; and pkgver is held to pacman's character set, because it lands in a file makepkg sources as shell. Ordering is vercmp's, not sort -V's -- they disagree about whether 1.0a precedes 1.0, and pacman is what decides if a published package is an upgrade. That is also why the workflow runs in an Arch container rather than straight on the runner. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/sync-upstream.yml | 95 +++++++++ README.md | 42 +++- bin/sync-upstream | 314 ++++++++++++++++++++++++++++ helpers/package-metadata.sh | 13 ++ 4 files changed, 462 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/sync-upstream.yml create mode 100755 bin/sync-upstream diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 0000000..0083ec3 --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,95 @@ +name: Sync Upstream Releases + +on: + schedule: + # Every 6 hours, off the hour to dodge the scheduling backlog at :00 + - cron: '20 */6 * * *' + workflow_dispatch: + inputs: + packages: + description: 'Specific packages to update (space-separated, leave empty for all)' + required: false + default: '' + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + # Runs in an Arch container for vercmp: whether a release is an upgrade has + # to be decided by the same comparator pacman will use on users' machines. + - name: Update packages from upstream release feeds + run: | + docker run --rm \ + -e PACKAGES="$PACKAGES" \ + -e HOST_UID="$(id -u)" \ + -e HOST_GID="$(id -g)" \ + -v "$PWD/bin:/workspace/bin:ro" \ + -v "$PWD/helpers:/workspace/helpers:ro" \ + -v "$PWD/pkgbuilds:/workspace/pkgbuilds" \ + -w /workspace \ + archlinux:base-devel bash -lc ' + set -euo pipefail + + pacman -Syu --noconfirm jq + + groupadd -g "$HOST_GID" runner + useradd -m -u "$HOST_UID" -g "$HOST_GID" runner + chown -R runner:runner /workspace/pkgbuilds + + if [[ -n "${PACKAGES:-}" ]]; then + read -r -a package_args <<< "$PACKAGES" + runuser -u runner -- ./bin/sync-upstream "${package_args[@]}" + else + runuser -u runner -- ./bin/sync-upstream + fi + ' + env: + PACKAGES: ${{ github.event.inputs.packages }} + + - name: Check for changes + id: changes + run: | + if [ -z "$(git status --porcelain)" ]; then + echo "has_changes=false" >> "$GITHUB_OUTPUT" + else + echo "has_changes=true" >> "$GITHUB_OUTPUT" + fi + + - name: Create Pull Request + if: steps.changes.outputs.has_changes == 'true' + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'chore: sync upstream releases' + title: 'chore: sync upstream releases' + body: | + Automated update of packages that track an upstream vendor release + feed rather than the AUR. + + Each package reports its newest release through + `.omarchy/upstream.sh`. + branch: auto/sync-upstream + delete-branch: true + labels: automated + reviewers: ryanrhughes + + - name: Notify Basecamp on failure + if: failure() && env.BASECAMP_CHATBOT_URL != '' + env: + BASECAMP_CHATBOT_URL: ${{ secrets.BASECAMP_CHATBOT_URL }} + run: | + curl -s -o /dev/null \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg content \ + "🔴 Upstream sync failed
View run" \ + '{content: $content}')" \ + "$BASECAMP_CHATBOT_URL" diff --git a/README.md b/README.md index 5f1e891..50c03f0 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ The filesystem no longer encodes release policy. Instead: - all other packages reach `stable` by promoting tested edge artifacts with `bin/repo migrate` - AUR sync behavior is controlled by `source`, `sync`, `aur`, patches, and hooks in `.omarchy/` - packages can opt out of unscoped builds with `skip_build`; explicit `--package` builds remain available +- packages that follow a vendor release feed instead of the AUR carry an `.omarchy/upstream.sh` hook ## Prerequisites ### aarch64 Builds (Optional) @@ -248,6 +249,40 @@ bin/sync-aur yay v4l2-relayd # Sync specific packages AUR sync is metadata-driven. It preserves `.omarchy/`, replaces the package root with AUR contents, applies `.omarchy/patches/*.patch`, runs `.omarchy/post-sync.sh` when present, applies pkgrel metadata, removes AUR-only `.SRCINFO` and `.gitignore` files, and records `upstream_commit`. +### Sync Upstream Releases + +```bash +bin/sync-upstream # Update every package with an upstream hook +bin/sync-upstream openai-codex-desktop # Update specific packages +``` + +Some vendors publish a release feed of their own that is faster and more precise +than the AUR packaging of it. Those packages are `source: local` — Omarchy owns +the PKGBUILD — and provide `.omarchy/upstream.sh`, a hook that reports the newest +upstream release as JSON on stdout: + +```json +{ + "pkgver": "1.2.3", + "sha256sums": { "x86_64": [""], "aarch64": [""] } +} +``` + +Architecture keys become `sha256sums_` in the PKGBUILD; the key `any` means +the unsuffixed `sha256sums` array, and only the arrays a hook names are touched. +An empty object (`{}`) reports no update, which is how a hook waits out a release +that has landed for one architecture but not yet the other. + +When the reported version is newer than the checked-in one, `bin/sync-upstream` +rewrites `pkgver` and those checksum arrays and resets `pkgrel` to 1. A version +that is equal or older leaves the package alone, so a vendor rolling a release +back cannot walk the repository backwards. + +Hooks should read checksums from whatever manifest the vendor publishes rather +than downloading the artifacts — see `pkgbuilds/openai-codex-desktop/.omarchy/upstream.sh`, +which reads OpenAI's Debian package index and never fetches the 750 MB of debs +it describes. + ### Other ```bash @@ -260,6 +295,7 @@ bin/repo push # Upload local builds to the host and publi bin/add-package # Add an AUR/local package with metadata bin/package-worktree # Create upstream/patched/current scratch workspace bin/repo remove # Remove package +bin/sync-upstream # Update packages that track a vendor release feed bin/clean-docker # Clear Docker images/cache (forces fresh rebuild) ``` @@ -345,7 +381,8 @@ omarchy-pkgs/ │ └── .omarchy/ │ ├── package.json # Source/sync/release metadata │ ├── patches/ # Omarchy patches reapplied after AUR sync -│ └── post-sync.sh # Optional dynamic post-sync customization hook +│ ├── post-sync.sh # Optional dynamic post-sync customization hook +│ └── upstream.sh # Optional vendor release feed hook (non-AUR packages) ├── build/ ├── build-output/ # Unsigned packages (temporary) │ ├── edge/ @@ -396,7 +433,7 @@ Minimal examples: Fields: -- `source`: `aur` or `local` +- `source`: `aur` or `local`. A `local` package can still follow an upstream release with an `.omarchy/upstream.sh` hook. - `sync`: optional for AUR packages; defaults to `true`. Set `false` for AUR-origin packages that Omarchy maintains manually. - `aur`: optional AUR package name when it differs from the local package directory, usually for split packages. - `release_ring`: optional. `fast` means the package is built directly for stable as well as edge. Packages without a ring build in edge and reach stable through tested artifact promotion (`bin/repo migrate`). @@ -539,6 +576,7 @@ The repository includes GitHub workflows and systemd services for automated rele #### GitHub Workflows 1. **sync-aur.yml** (Every 6 hours): Syncs AUR packages according to `.omarchy/package.json` and opens a PR when changes are found. +2. **sync-upstream.yml** (Every 6 hours): Runs `.omarchy/upstream.sh` for packages that track a vendor release feed and opens a PR when a newer version is out. #### Systemd Services diff --git a/bin/sync-upstream b/bin/sync-upstream new file mode 100755 index 0000000..c307cf8 --- /dev/null +++ b/bin/sync-upstream @@ -0,0 +1,314 @@ +#!/bin/bash +set -euo pipefail + +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +source "$BUILD_ROOT/helpers/message-helpers.sh" +source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/package-metadata.sh" + +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT + +SPECIFIC_PACKAGES=() + +usage() { + cat </.omarchy/upstream.sh, a hook +that reports the newest upstream release as JSON on stdout: + + { + "pkgver": "1.2.3", + "sha256sums": { "x86_64": [""], "aarch64": [""] } + } + +Architecture keys become sha256sums_ in the PKGBUILD; the key "any" means +the unsuffixed sha256sums array. An empty object ({}) reports no update. + +When the reported version is newer than the checked-in one, pkgver and the +listed checksum arrays are rewritten and pkgrel is reset to 1. + +Arguments: + PACKAGE One or more package names to update (optional) + +Examples: + $0 # Update every package with an upstream hook + $0 openai-codex-desktop # Update specific packages +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --*) + print_error "Unknown option: $1" + exit 1 + ;; + *) + SPECIFIC_PACKAGES+=("$1") + shift + ;; + esac +done + +if ! command -v vercmp >/dev/null 2>&1; then + print_error "vercmp not found: this needs pacman to decide whether a release is an upgrade" + exit 1 +fi + +print_header "Upstream Package Sync" + +UPDATED=0 +SKIPPED=0 +FAILED=0 +SPECIFIC_MODE=false + +get_pkgver() { + local package_dir="$1" + + grep -m1 '^pkgver=' "$package_dir/PKGBUILD" | cut -d= -f2- | tr -d "\"'" +} + +assert_single_assignment() { + local pkgbuild="$1" + local pattern="$2" + local label="$3" + + if [[ $(grep -c "$pattern" "$pkgbuild") -ne 1 ]]; then + print_error "Expected exactly one $label assignment in $pkgbuild" + return 1 + fi +} + +set_pkgbuild_scalar() { + local pkgbuild="$1" + local field="$2" + local value="$3" + + assert_single_assignment "$pkgbuild" "^${field}=" "$field" || return 1 + sed -i "s/^${field}=.*/${field}=${value}/" "$pkgbuild" +} + +# Replace an array assignment, however many lines the original spans. +set_pkgbuild_array() { + local pkgbuild="$1" + local name="$2" + shift 2 + local values=("$@") + + assert_single_assignment "$pkgbuild" "^${name}=(" "$name" || return 1 + + local block="$TEMP_DIR/array-block" + if [[ ${#values[@]} -eq 1 ]]; then + printf "%s=('%s')\n" "$name" "${values[0]}" > "$block" + else + printf '%s=(\n' "$name" > "$block" + printf " '%s'\n" "${values[@]}" >> "$block" + printf ')\n' >> "$block" + fi + + # Rewrite beside the PKGBUILD so the move is an atomic same-filesystem rename. + local rewritten="$pkgbuild.sync-upstream" + if ! awk -v prefix="${name}=(" -v block="$block" ' + !replaced && index($0, prefix) == 1 { + while ((getline line < block) > 0) print line + close(block) + replaced = 1 + # A ")" anywhere past the opening closes the array; testing for one at end + # of line instead would treat a trailing comment as a continuation and eat + # every line up to the next ")". + if (index(substr($0, length(prefix) + 1), ")") == 0) skipping = 1 + next + } + skipping { if ($0 ~ /\)/) skipping = 0; next } + { print } + ' "$pkgbuild" > "$rewritten"; then + print_error "Failed to rewrite ${name} in $pkgbuild" + rm -f "$rewritten" + return 1 + fi + + chmod --reference="$pkgbuild" "$rewritten" + mv "$rewritten" "$pkgbuild" +} + +# pacman's own comparator, because nothing else agrees with it at the corners: +# sort -V calls 1.0a newer than 1.0, vercmp calls it older, and pacman is what +# decides whether a published package is an upgrade. +version_is_newer() { + local candidate="$1" + local current="$2" + + [[ "$candidate" != "$current" ]] || return 1 + [[ "$(vercmp "$candidate" "$current")" -gt 0 ]] +} + +validate_release() { + local release="$1" + + # pkgver is written into the PKGBUILD, which makepkg sources as shell, so it + # is held to pacman's own character set rather than merely being non-empty. + jq -e ' + (.pkgver | type == "string" and test("^[A-Za-z0-9._+]+$")) + and (.sha256sums | type == "object" and length > 0) + and (.sha256sums | to_entries | all( + .key | test("^[a-z0-9_]+$") + )) + and (.sha256sums | to_entries | all( + .value | type == "array" and length > 0 and all(test("^[0-9a-f]{64}$")) + )) + ' <<<"$release" >/dev/null +} + +apply_release() { + local package_dir="$1" + local release="$2" + local pkgver="$3" + local pkgbuild="$package_dir/PKGBUILD" + + local arch array values + local targets=() + + while IFS= read -r arch; do + if [[ "$arch" == "any" ]]; then + array="sha256sums" + else + array="sha256sums_$arch" + fi + targets+=("$arch:$array") + done < <(jq -r '.sha256sums | keys[]' <<<"$release") + + # Everything the update will touch is checked before anything is written. A + # hook naming an array the PKGBUILD does not have must fail with the file + # untouched rather than half rewritten. + assert_single_assignment "$pkgbuild" '^pkgver=' pkgver || return 1 + assert_single_assignment "$pkgbuild" '^pkgrel=' pkgrel || return 1 + local target + for target in "${targets[@]}"; do + assert_single_assignment "$pkgbuild" "^${target#*:}=(" "${target#*:}" || return 1 + done + + for target in "${targets[@]}"; do + arch="${target%%:*}" + array="${target#*:}" + + mapfile -t values < <(jq -r --arg arch "$arch" '.sha256sums[$arch][]' <<<"$release") + set_pkgbuild_array "$pkgbuild" "$array" "${values[@]}" || return 1 + done + + set_pkgbuild_scalar "$pkgbuild" pkgver "$pkgver" || return 1 + set_pkgbuild_scalar "$pkgbuild" pkgrel 1 || return 1 +} + +sync_package() { + local package="$1" + local package_dir="$PKGBUILDS_DIR/$package" + local hook="$package_dir/.omarchy/upstream.sh" + + if [[ ! -f "$package_dir/PKGBUILD" ]]; then + print_error "Package $package has no PKGBUILD" + ((++FAILED)) + return 0 + fi + + if [[ ! -f "$hook" ]]; then + if [[ "$SPECIFIC_MODE" == true ]]; then + print_error "Package $package is missing .omarchy/upstream.sh" + ((++FAILED)) + else + print_info "Skipping $package: no upstream hook" + ((++SKIPPED)) + fi + return 0 + fi + + print_info "Checking $package for upstream releases..." + + local release + if ! release=$(cd "$package_dir" && PACKAGE_NAME="$package" bash .omarchy/upstream.sh); then + print_error "Upstream hook failed for $package" + ((++FAILED)) + return 0 + fi + + if ! jq -e . >/dev/null 2>&1 <<<"$release"; then + print_error "Upstream hook for $package did not report valid JSON" + ((++FAILED)) + return 0 + fi + + if [[ "$(jq -r 'has("pkgver")' <<<"$release")" == "false" ]]; then + print_info " No upstream update reported" + ((++SKIPPED)) + return 0 + fi + + if ! validate_release "$release"; then + print_error "Upstream hook for $package reported a malformed release" + ((++FAILED)) + return 0 + fi + + local pkgver current_pkgver + pkgver=$(jq -r '.pkgver' <<<"$release") + current_pkgver=$(get_pkgver "$package_dir") + + if [[ -z "$current_pkgver" ]]; then + print_error "Could not read pkgver from $package_dir/PKGBUILD" + ((++FAILED)) + return 0 + fi + + if [[ "$pkgver" == "$current_pkgver" ]]; then + print_info " Already at $current_pkgver" + ((++SKIPPED)) + return 0 + fi + + if ! version_is_newer "$pkgver" "$current_pkgver"; then + print_warning " Upstream reports $pkgver, older than the checked-in $current_pkgver; leaving it alone" + ((++SKIPPED)) + return 0 + fi + + if ! apply_release "$package_dir" "$release" "$pkgver"; then + print_error "Failed to update $package" + ((++FAILED)) + return 0 + fi + + print_success " $current_pkgver -> $pkgver" + ((++UPDATED)) +} + +if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then + SPECIFIC_MODE=true + for package in "${SPECIFIC_PACKAGES[@]}"; do + sync_package "$package" + done +else + while IFS= read -r package; do + sync_package "$package" + done < <(packages_for_upstream_sync) +fi + +echo "" +if [[ $FAILED -gt 0 ]]; then + print_error "Upstream sync completed with failures" +else + print_success "Upstream sync complete!" +fi +echo " Target: $PKGBUILDS_DIR" +echo " Updated: $UPDATED" +echo " Skipped: $SKIPPED" +echo " Failed: $FAILED" + +if [[ $FAILED -gt 0 ]]; then + exit 1 +fi diff --git a/helpers/package-metadata.sh b/helpers/package-metadata.sh index 1160fff..b1f1b70 100644 --- a/helpers/package-metadata.sh +++ b/helpers/package-metadata.sh @@ -135,6 +135,19 @@ packages_for_aur_sync() { done } +package_has_upstream_hook() { + local pkgdir="$1" + [[ -f "$pkgdir/.omarchy/upstream.sh" ]] +} + +packages_for_upstream_sync() { + package_dirs | while IFS= read -r pkgdir; do + if package_has_upstream_hook "$pkgdir"; then + basename "$pkgdir" + fi + done +} + packages_for_mirror() { local mirror="$1"