From fc2a8aeed06f3f78680b91135ddcd58450227975 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Fri, 25 Sep 2026 17:45:42 +1000 Subject: [PATCH 1/2] Detect aarch64 rebuilds, including against carried dependencies sync-rebuilds followed PUBLISHED_ARCHES (x86_64 by default), so the scheduled run skipped every aarch64-only package, and it read aarch64 triggers from Arch Linux ARM even when this repository carries them. It now follows the architectures a merge builds (CI_ARCHES, shared with build-matrix), and reads a carried trigger from its recipe once edge publishes that version; until then its dependents wait. Published databases are verified before use and loaded once per run. --- README.md | 7 +- bin/build-matrix | 2 +- bin/sync-rebuilds | 201 +++++++++++++++++++++++++++++++++++++++------- helpers/paths.sh | 14 ++++ 4 files changed, 191 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8823595..6920a14 100644 --- a/README.md +++ b/README.md @@ -520,7 +520,8 @@ A package names those dependencies in `.omarchy/package.json`: ``` `bin/sync-rebuilds` reads each named package's version from the official -repositories for every published architecture the package supports and compares +repositories for every architecture a merge builds (`CI_ARCHES` in +`helpers/paths.sh`, both by default) that the package supports and compares it to `rebuilt_against`. Records are kept per architecture because Arch and Arch Linux ARM can carry different dependency versions. pkgrel is bumped once when any recorded version moves; that one source revision is then rebuilt by @@ -539,6 +540,8 @@ repository database, which is also what the ARM builder uses. Testing and staging repositories do not count. A legacy flat `rebuilt_against` record is read as x86_64 and is migrated naturally the next time a rebuild is needed. +A dependency this repository carries for an architecture (a recipe here that builds for edge on it, such as aquamarine on aarch64) shadows the distribution's, because the builder lists `[omarchy]` first. Its version is the recipe's, and it counts only once edge publishes that version: until then the builder still links against the previous one, so dependents are left alone for that run. + ### Other ```bash @@ -726,7 +729,7 @@ Fields: - `skip_build`: optional boolean; defaults to `false`. Set `true` to exclude a package from scheduled version checks and unscoped builds. The package can still be built explicitly with `bin/repo release --package `. - `pkgrel`: legacy import customization metadata. Maintained recipes keep their complete package release directly in PKGBUILD; rebuilds increment it there. - `rebuild_on`: optional array of package names this package links against closely enough that it must be rebuilt when they change, independent of its own source. Read by `bin/sync-rebuilds`. -- `rebuilt_against`: written by `bin/sync-rebuilds`. Maps each published architecture to the versions of its `rebuild_on` packages that the current pkgrel was bumped for. +- `rebuilt_against`: written by `bin/sync-rebuilds`. Maps each built architecture to the versions of its `rebuild_on` packages that the current pkgrel was bumped for. - `upstream_commit`: legacy AUR metadata, superseded by `origin.commit`. `bin/package-worktree` can use historical provenance to inspect the original recipe. ### Build Matrix diff --git a/bin/build-matrix b/bin/build-matrix index e772349..584a3bd 100755 --- a/bin/build-matrix +++ b/bin/build-matrix @@ -19,7 +19,7 @@ BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") source "$BUILD_ROOT/helpers/paths.sh" source "$BUILD_ROOT/helpers/package-metadata.sh" -ARCHES=${CI_ARCHES:-x86_64 aarch64} +ARCHES=$CI_ARCHES if [[ "${1:-}" == --arch ]]; then [[ $2 == all ]] || ARCHES=$2; shift 2; fi for a in $ARCHES; do require_valid_arch "$a"; done diff --git a/bin/sync-rebuilds b/bin/sync-rebuilds index 3009b89..d99277f 100755 --- a/bin/sync-rebuilds +++ b/bin/sync-rebuilds @@ -67,8 +67,10 @@ Examples: $0 quickshell-git # Update specific packages x86_64 trigger versions come from the local pacman database; aarch64 versions -come from the live Arch Linux ARM repository database. Only architectures in -PUBLISHED_ARCHES that the package supports are considered. +come from the live Arch Linux ARM repository database. A trigger this +repository carries for an architecture shadows both, so its version is the +checked-in recipe's once edge publishes it. Every architecture in CI_ARCHES +(what a merge builds) that the package supports is considered. EOF } @@ -167,6 +169,8 @@ repo_version() { # repo_version } declare -A PUBLISHED_VERSION=() +declare -A EDGE_VERSION=() +declare -A EDGE_READ=() PUBLISHED_LOADED=false remember_published() { @@ -190,7 +194,7 @@ load_published_versions() { local arch mirror db name base version - for arch in $(published_arches); do + for arch in $(ci_arches); do for mirror in "${PUBLISHED_MIRRORS[@]}"; do db="$TEMP_DIR/published-$mirror-$arch.db.tar.zst" @@ -199,10 +203,19 @@ load_published_versions() { print_warning "Could not read the published $mirror/$arch database; bumps are not checked against it this run" continue fi + if ! tar -tf "$db" >/dev/null 2>&1; then + print_warning "Unreadable published $mirror/$arch database; bumps are not checked against it this run" + continue + fi + [[ "$mirror" == edge ]] && EDGE_READ["$arch"]=1 while IFS=$'\t' read -r name base version; do [[ -n "$name" && -n "$version" ]] && remember_published "$name" "$version" [[ -n "$base" && -n "$version" ]] && remember_published "$base" "$version" + if [[ "$mirror" == edge && -n "$version" ]]; then + [[ -z "$name" ]] || EDGE_VERSION["$arch/$name"]="$version" + [[ -z "$base" ]] || EDGE_VERSION["$arch/$base"]="$version" + fi done < <( tar -xOf "$db" --wildcards '*/desc' 2>/dev/null | awk ' function emit() { @@ -227,6 +240,27 @@ published_version() { echo "${PUBLISHED_VERSION[$package]:-}" } +# A trigger this repository builds for the architecture. The builder lists +# [omarchy] ahead of the distribution repositories, so this recipe, not Arch or +# Arch Linux ARM, decides what a dependent links against. +carried_trigger_dir() { # carried_trigger_dir + local pkgdir + pkgdir=$(package_dir_for_name "$2") || return 1 + package_has_metadata "$pkgdir" || return 1 + package_builds_for_mirror "$pkgdir" edge || return 1 + package_supports_arch "$pkgdir" "$1" || return 1 + echo "$pkgdir" +} + +carried_version() { # carried_version + local epoch pkgver pkgrel + epoch=$(package_pkgbuild_var "$2" epoch "$1") || return 1 + pkgver=$(package_pkgbuild_var "$2" pkgver "$1") || return 1 + pkgrel=$(package_pkgbuild_var "$2" pkgrel "$1") || return 1 + [[ -n "$pkgver" && -n "$pkgrel" ]] || return 1 + echo "${epoch:+$epoch:}$pkgver-$pkgrel" +} + # The pkgver of a full version string, with any epoch and pkgrel removed. version_pkgver() { local version="${1#*:}" @@ -368,13 +402,33 @@ sync_package() { print_info "Checking $package against ${triggers[*]}..." - local current="{}" arch arch_current trigger version considered=0 - for arch in $(published_arches); do + local current="{}" arch arch_current trigger version carried considered=0 + for arch in $(ci_arches); do package_supports_arch "$package_dir" "$arch" || continue considered=$((considered + 1)) arch_current="{}" for trigger in "${triggers[@]}"; do - if ! version=$(repo_version "$arch" "$trigger"); then + if carried=$(carried_trigger_dir "$arch" "$trigger"); then + if ! version=$(carried_version "$arch" "$carried"); then + print_error " Could not read the $arch version of $trigger from its recipe; leaving $package alone" + ((++FAILED)) + return 0 + fi + load_published_versions + if [[ -z "${EDGE_READ[$arch]:-}" ]]; then + print_error " Could not read the published edge/$arch database for $trigger; leaving $package alone" + ((++FAILED)) + return 0 + fi + # Until edge publishes the recipe's version the builder still links + # against the previous one, and recording the new version now would + # certify a build that never saw it. + if [[ "${EDGE_VERSION[$arch/$trigger]:-}" != "$version" ]]; then + print_warning " $trigger $version is not published on edge/$arch yet; leaving $package alone until it is" + ((++SKIPPED)) + return 0 + fi + elif ! version=$(repo_version "$arch" "$trigger"); then print_error " Could not read $arch repository versions; leaving $package alone" ((++FAILED)) return 0 @@ -479,6 +533,7 @@ sync_package() { # checkout that has fallen behind the repository can otherwise be bumped to # something pacman orders below what it would replace. local floor + load_published_versions floor=$(published_version "$package") if [[ -n "$floor" && "$(version_pkgver "$floor")" == "$pkgver" ]]; then if [[ "$(vercmp "$new_version" "$floor")" -le 0 ]]; then @@ -584,26 +639,33 @@ STUB chmod +x "$root/stub/pacman" } -# Serves a repository database assembled by hand from name=version pairs. With -# none given the stub fails, which is how the unreachable-repository path is +# A repository database assembled by hand from name=version pairs. +selftest_db() { + local out="$1" + shift + local staging="$out.d" entry name version + + rm -rf "$staging" + mkdir -p "$staging" + for entry in "$@"; do + name="${entry%=*}" + version="${entry#*=}" + mkdir -p "$staging/$name-$version" + printf '%%FILENAME%%\n%s-%s-x86_64.pkg.tar.zst\n\n%%NAME%%\n%s\n\n%%BASE%%\n%s\n\n%%VERSION%%\n%s\n' \ + "$name" "$version" "$name" "$name" "$version" > "$staging/$name-$version/desc" + done + tar --zstd -cf "$out" -C "$staging" . +} + +# Serves one published database for every channel and architecture. With no +# pairs given the stub fails, which is how the unreachable-repository path is # exercised. selftest_published() { local root="$1" shift - local staging="$root/stub/db" - local entry name version if [[ $# -gt 0 ]]; then - rm -rf "$staging" - mkdir -p "$staging" - for entry in "$@"; do - name="${entry%=*}" - version="${entry#*=}" - mkdir -p "$staging/$name-$version" - printf '%%FILENAME%%\n%s-%s-x86_64.pkg.tar.zst\n\n%%NAME%%\n%s\n\n%%BASE%%\n%s\n\n%%VERSION%%\n%s\n' \ - "$name" "$version" "$name" "$name" "$version" > "$staging/$name-$version/desc" - done - tar --zstd -cf "$root/stub/omarchy.db.tar.zst" -C "$staging" . + selftest_db "$root/stub/omarchy.db.tar.zst" "$@" fi cat > "$root/stub/curl" <<'STUB' @@ -616,19 +678,33 @@ while [[ $# -gt 0 ]]; do *) url="$1"; shift ;; esac done -if [[ "$url" == */aarch64/* ]]; then - repo="${url%/*}" - repo="${repo##*/}" - db="$(dirname "$0")/alarm-$repo.db" -else - db="$(dirname "$0")/omarchy.db.tar.zst" -fi +case "$url" in + */omarchy.db.tar.zst) + channel="${url%/omarchy.db.tar.zst}" + arch="${channel##*/}" + channel="${channel%/*}" + db="$(dirname "$0")/omarchy-${channel##*/}-$arch.db.tar.zst" + [[ -f "$db" ]] || db="$(dirname "$0")/omarchy.db.tar.zst" + ;; + */aarch64/*) + repo="${url%/*}" + db="$(dirname "$0")/alarm-${repo##*/}.db" + ;; + *) db="" ;; +esac [[ -f "$db" && -n "$out" ]] || exit 22 cp "$db" "$out" STUB chmod +x "$root/stub/curl" } +# Serves one channel and architecture its own published database. +selftest_channel() { # selftest_channel [name=version...] + local root="$1" channel="$2" arch="$3" + shift 3 + selftest_db "$root/stub/omarchy-$channel-$arch.db.tar.zst" "$@" +} + selftest_alarm() { local root="$1" shift @@ -668,12 +744,15 @@ cmd_self_test() { fi } + # SELFTEST_ARCHES stands in for CI_ARCHES; "default" leaves it unset, as the + # scheduled workflow does. run_case() { local root="$1" shift - local status=0 - OMARCHY_ARCHES="${SELFTEST_ARCHES:-x86_64}" \ - PATH="$root/stub:$PATH" "$root/bin/sync-rebuilds" "$@" > "$root/output" 2>&1 || status=$? + local status=0 arches=(CI_ARCHES="${SELFTEST_ARCHES:-x86_64}") + [[ "${SELFTEST_ARCHES:-}" != default ]] || arches=(-u CI_ARCHES) + env "${arches[@]}" PATH="$root/stub:$PATH" \ + "$root/bin/sync-rebuilds" "$@" > "$root/output" 2>&1 || status=$? echo "$status" } @@ -781,6 +860,68 @@ cmd_self_test() { unset SELFTEST_ARCHES check "pkgrel untouched" 1 "$(pkgrel_of "$root/pkgbuilds/t-x86")" + echo "An ARM-only package is checked when no architecture list is given:" + root=$(selftest_root arm-default) + selftest_package "$root" t-arm-default 1 '{"source":"local","rebuild_on":["dep-a"],"rebuilt_against":{"aarch64":{"dep-a":"1-1"}}}' 1.0 aarch64 + selftest_pacman "$root" + selftest_published "$root" + selftest_alarm "$root" dep-a=2-1 + SELFTEST_ARCHES=default + check "run succeeds" 0 "$(run_case "$root")" + unset SELFTEST_ARCHES + check "pkgrel bumped" 2 "$(pkgrel_of "$root/pkgbuilds/t-arm-default")" + check "ARM trigger recorded" "2-1" \ + "$(jq -r '.rebuilt_against.aarch64["dep-a"]' "$root/pkgbuilds/t-arm-default/.omarchy/package.json")" + + echo "A dependency carried here for ARM is read from its recipe once edge publishes it:" + root=$(selftest_root carried) + selftest_package "$root" t-carried 1.1 '{"source":"local","channels":["edge"]}' 0.15.1 aarch64 + selftest_package "$root" t-linked 3 '{"source":"local","rebuild_on":["t-carried"],"rebuilt_against":{"x86_64":{"t-carried":"0.15.1-1"},"aarch64":{"t-carried":"0.15.0-2"}}}' 1.0 'x86_64 aarch64' + selftest_pacman "$root" t-carried=0.15.1-1 + selftest_published "$root" + selftest_alarm "$root" t-carried=0.15.1-1 + selftest_channel "$root" edge aarch64 t-carried=0.15.1-1.1 + SELFTEST_ARCHES=default + check "run succeeds" 0 "$(run_case "$root")" + check "pkgrel bumped" 4 "$(pkgrel_of "$root/pkgbuilds/t-linked")" + check "the carried version is recorded, not Arch Linux ARM's" "0.15.1-1.1" \ + "$(jq -r '.rebuilt_against.aarch64["t-carried"]' "$root/pkgbuilds/t-linked/.omarchy/package.json")" + check "x86_64, where nothing is carried, still reads the distribution" "0.15.1-1" \ + "$(jq -r '.rebuilt_against.x86_64["t-carried"]' "$root/pkgbuilds/t-linked/.omarchy/package.json")" + check "a second run succeeds" 0 "$(run_case "$root")" + unset SELFTEST_ARCHES + check "a second run leaves it alone" 4 "$(pkgrel_of "$root/pkgbuilds/t-linked")" + + echo "A carried dependency that edge does not publish yet leaves its dependents alone:" + root=$(selftest_root carried-in-flight) + selftest_package "$root" t-carried 1.2 '{"source":"local","channels":["edge"]}' 0.15.1 aarch64 + selftest_package "$root" t-linked 4 '{"source":"local","rebuild_on":["t-carried"],"rebuilt_against":{"aarch64":{"t-carried":"0.15.1-1.1"}}}' 1.0 aarch64 + selftest_pacman "$root" + selftest_published "$root" + selftest_alarm "$root" t-carried=0.15.1-1 + selftest_channel "$root" edge aarch64 t-carried=0.15.1-1.1 + SELFTEST_ARCHES=aarch64 + check "run succeeds" 0 "$(run_case "$root")" + unset SELFTEST_ARCHES + check "pkgrel untouched" 4 "$(pkgrel_of "$root/pkgbuilds/t-linked")" + check "record untouched" "0.15.1-1.1" \ + "$(jq -r '.rebuilt_against.aarch64["t-carried"]' "$root/pkgbuilds/t-linked/.omarchy/package.json")" + + echo "A carried dependency whose edge database cannot be read fails the run:" + root=$(selftest_root carried-unreadable) + selftest_package "$root" t-carried 1.1 '{"source":"local","channels":["edge"]}' 0.15.1 aarch64 + selftest_package "$root" t-linked 3 '{"source":"local","rebuild_on":["t-carried"],"rebuilt_against":{"aarch64":{"t-carried":"0.15.0-2"}}}' 1.0 aarch64 + selftest_pacman "$root" + selftest_published "$root" + selftest_alarm "$root" t-carried=0.15.1-1 + SELFTEST_ARCHES=aarch64 + check "run fails" 1 "$(run_case "$root")" + check "pkgrel untouched" 3 "$(pkgrel_of "$root/pkgbuilds/t-linked")" + printf 'not a database\n' > "$root/stub/omarchy-edge-aarch64.db.tar.zst" + check "a corrupt download fails the run too" 1 "$(run_case "$root")" + unset SELFTEST_ARCHES + check "pkgrel still untouched" 3 "$(pkgrel_of "$root/pkgbuilds/t-linked")" + echo "A PKGBUILD that branches on CARCH at file scope still reads its version:" root=$(selftest_root carch-branch) mkdir -p "$root/pkgbuilds/t-carch/.omarchy" diff --git a/helpers/paths.sh b/helpers/paths.sh index 5d6191c..1d734a9 100644 --- a/helpers/paths.sh +++ b/helpers/paths.sh @@ -43,6 +43,20 @@ reference_arch() { published_arches | head -1 } +# Architectures a merge to master builds and publishes: bin/build-matrix plans +# a PR build for each one a package supports, and publish.yml ships them. This +# is independent of PUBLISHED_ARCHES, so anything deciding what a merge has to +# rebuild (bin/sync-rebuilds) follows this list. CI_ARCHES overrides it. +CI_ARCHES="${CI_ARCHES:-x86_64 aarch64}" + +ci_arches() { + local arch + for arch in $CI_ARCHES; do + require_valid_arch "$arch" + echo "$arch" + done +} + # Scheduled-pipeline state, one file per channel and architecture, so one # architecture's queue or backoff never gates another's. STATE_DIR="${OMARCHY_STATE_DIR:-/root/.state}" From 33ab3d2bf8b8e89df2fa5a8817493af135aefc74 Mon Sep 17 00:00:00 2001 From: Marcelo Alcantara Date: Fri, 25 Sep 2026 17:51:54 +1000 Subject: [PATCH 2/2] Spell a carried version without a zero epoch, as makepkg publishes it --- bin/sync-rebuilds | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bin/sync-rebuilds b/bin/sync-rebuilds index d99277f..32fc775 100755 --- a/bin/sync-rebuilds +++ b/bin/sync-rebuilds @@ -258,6 +258,8 @@ carried_version() { # carried_version pkgver=$(package_pkgbuild_var "$2" pkgver "$1") || return 1 pkgrel=$(package_pkgbuild_var "$2" pkgrel "$1") || return 1 [[ -n "$pkgver" && -n "$pkgrel" ]] || return 1 + # makepkg leaves a zero epoch out of the published version + [[ "$epoch" != 0 ]] || epoch="" echo "${epoch:+$epoch:}$pkgver-$pkgrel" } @@ -942,6 +944,15 @@ PKG check "arch-specific value follows CARCH" "arm64" "$(package_pkgbuild_var "$root/pkgbuilds/t-carch" _suffix aarch64)" check "unsupported arch reports failure" 1 "$(package_pkgbuild_var "$root/pkgbuilds/t-carch" pkgver armv7h >/dev/null; echo $?)" + echo "A carried recipe's version is spelled the way makepkg publishes it:" + root=$(selftest_root carried-epoch) + selftest_package "$root" t-epoch 1.1 '{"source":"local"}' 0.15.1 aarch64 + check "no epoch" "0.15.1-1.1" "$(carried_version aarch64 "$root/pkgbuilds/t-epoch")" + echo 'epoch=0' >> "$root/pkgbuilds/t-epoch/PKGBUILD" + check "a zero epoch is left out" "0.15.1-1.1" "$(carried_version aarch64 "$root/pkgbuilds/t-epoch")" + echo 'epoch=2' >> "$root/pkgbuilds/t-epoch/PKGBUILD" + check "a real epoch is kept" "2:0.15.1-1.1" "$(carried_version aarch64 "$root/pkgbuilds/t-epoch")" + echo "" if [[ "$failures" -eq 0 ]]; then print_success "Self-test passed"