Make aarch64 a first-class architecture in the scheduled pipeline

One list, PUBLISHED_ARCHES in helpers/paths.sh (default x86_64,
overridable with OMARCHY_ARCHES), now drives everything the repository
host schedules. check-versions compares PKGBUILDs against each
architecture's channel databases and writes one queue per channel and
architecture; auto-release works through the queues one architecture at
a time, each with its own backoff, so a failing build on one never
blocks the other; advance-channel --arch all re-runs an advance for every
published architecture and omarchy-release uses it for start and ship,
building the pinned pair once per architecture in its rc trigger; the
train observes channels through the reference (first) architecture
instead of a hard-coded x86_64. Queue and backoff files written under
the old per-channel names are treated as x86_64 until consumed.

Two things made an aarch64 builder image impossible to create: the
keyring bootstrap fetched omarchy-keyring from the target architecture's
own channel tree, which does not exist before that architecture has
published anything, and the QEMU probe only knew the x86_64-host,
aarch64-target case. The keyring (arch=any) now always comes from the
x86_64 tree, and the probe compares host and target architectures and
runs a container for the target platform.

clean-repo grouped versions with a regex that only knew any, x86_64 and
i686, so aarch64 packages would never have been pruned.
This commit is contained in:
Marcelo Alcantara authored and Ryan Hughes committed 2026-09-04 23:40:49 -04:00
1 parent 069ffc8c3a
commit 91843ab099
12 files changed
+330 -104

No files matched your search

+42 -1
View File
@@ -26,6 +26,32 @@ FAST_RING_ONLY=false
BOOTSTRAP=false
PACKAGES=""
# Kept for --arch all, which re-invokes this script per architecture with the
# other arguments unchanged (minus the --arch all pair itself).
ORIGINAL_ARGS=()
skip_next=false
prev=""
for arg in "$@"; do
if [[ "$skip_next" == true ]]; then
skip_next=false
prev=""
continue
fi
if [[ "$arg" == "--arch" ]]; then
prev="--arch"
continue
fi
if [[ "$prev" == "--arch" ]]; then
prev=""
if [[ "$arg" == "all" ]]; then
continue
fi
ORIGINAL_ARGS+=("--arch" "$arg")
continue
fi
ORIGINAL_ARGS+=("$arg")
done
usage() {
echo "Usage: $0 --from <channel> --to <channel> [OPTIONS]"
echo ""
@@ -36,7 +62,8 @@ usage() {
echo " or --bootstrap (one-time initial seed)"
echo ""
echo "Options:"
echo " --arch <arch> Target architecture (x86_64 or aarch64, default: x86_64)"
echo " --arch <arch>|all Target architecture (x86_64 or aarch64, default: x86_64);"
echo " all = every published architecture, one after another"
echo " --package <names...> Advance only the named package(s)"
echo " --fast-ring Restrict to fast-ring packages (stable -> rc parity copy)"
echo " --bootstrap One-time stable -> rc seed before forward-only enforcement"
@@ -65,6 +92,20 @@ while [[ $# -gt 0 ]]; do
shift 2
;;
--arch)
if [[ "$2" == "all" ]]; then
# A release train advances every published architecture together; a
# channel that moved for one and not another would serve mismatched
# versions to users on the two. Re-run this script once per
# architecture with the same arguments, stopping at the first failure
# so the operator sees which architecture is stuck (the advance is
# idempotent, so re-running with --arch all resumes where it stopped).
shift 2
for arch in $(published_arches); do
"$0" --arch "$arch" "${ORIGINAL_ARGS[@]}" || exit $?
done
exit 0
fi
require_valid_arch "$2"
ARCH="$2"
update_arch_paths
shift 2
+93 -57
View File
@@ -1,6 +1,10 @@
#!/bin/bash
# Run the release workflow for a channel when work is queued.
# Usage: auto-release <edge|rc|stable>
# Usage: auto-release <edge|rc|stable> [<arch>|all]
#
# With no architecture (what the timers pass), every published architecture
# is processed in turn, each against its own queue and its own backoff, so a
# failing aarch64 build never holds up x86_64 or the other way round.
#
# Safe to run on a tight schedule. Three guards make that true:
#
@@ -21,7 +25,7 @@ source "$BUILD_ROOT/helpers/paths.sh"
source "$BUILD_ROOT/helpers/lock-helpers.sh"
MIRROR="${1:-}"
STATE_DIR="${OMARCHY_STATE_DIR:-/root/.state}"
ARCH_ARG="${2:-all}"
# Backoff schedule: 10m, 20m, 40m, 80m, 160m, 320m, then hourly-ish forever
# (capped at 6h, the cadence this system ran at before frequent timers).
@@ -29,8 +33,9 @@ BACKOFF_BASE_SECONDS="${OMARCHY_BACKOFF_BASE:-600}"
BACKOFF_MAX_SECONDS="${OMARCHY_BACKOFF_MAX:-21600}"
if [[ -z "$MIRROR" ]]; then
print_error "Usage: $0 <mirror>"
print_error "Usage: $0 <mirror> [<arch>|all]"
echo " mirror: edge, rc, or stable"
echo " arch: one of $VALID_ARCHES, or all (default) for every published architecture"
exit 1
fi
@@ -39,17 +44,13 @@ if [[ "$MIRROR" != "edge" && "$MIRROR" != "rc" && "$MIRROR" != "stable" ]]; then
exit 1
fi
STATE_FILE="$STATE_DIR/.sync-needed-$MIRROR"
FAIL_FILE="$STATE_DIR/.build-failed-$MIRROR"
# Nothing queued: stay quiet. At a 5-minute cadence this is most invocations,
# and a header for each would bury the runs that matter in the journal.
if [[ ! -f "$STATE_FILE" ]]; then
exit 0
if [[ "$ARCH_ARG" == "all" ]]; then
ARCHES=$(published_arches)
else
require_valid_arch "$ARCH_ARG"
ARCHES="$ARCH_ARG"
fi
print_header "Processing Sync for $MIRROR"
# The build inputs are this checkout's contents; its HEAD identifies them.
current_fingerprint() {
git -C "$BUILD_ROOT" rev-parse HEAD 2>/dev/null || echo "unknown"
@@ -65,57 +66,92 @@ backoff_seconds() {
echo "$delay"
}
FAIL_COUNT=0
if [[ -f "$FAIL_FILE" ]]; then
# shellcheck disable=SC1090
source "$FAIL_FILE" 2>/dev/null || true
FAIL_COUNT="${FAILURE_COUNT:-0}"
failed_at="${FAILURE_AT:-0}"
failed_fingerprint="${FAILURE_FINGERPRINT:-}"
# Returns 0 when this architecture's queue was processed (or was empty), 1 when
# the release failed. Backoff and "someone else holds the lock" are not
# failures: they are this tick deciding to do nothing.
release_arch() {
local arch="$1"
local state_file fail_file
state_file=$(sync_queue_file "$MIRROR" "$arch")
fail_file=$(sync_fail_file "$MIRROR" "$arch")
if [[ "$failed_fingerprint" != "$(current_fingerprint)" ]]; then
print_info "Repository changed since the last failure — clearing backoff and retrying"
rm -f "$FAIL_FILE"
FAIL_COUNT=0
else
delay=$(backoff_seconds "$FAIL_COUNT")
now=$(date +%s)
retry_at=$((failed_at + delay))
if ((now < retry_at)); then
print_warning "$MIRROR has failed $FAIL_COUNT time(s) on this tree — not retrying until $(date -d "@$retry_at" '+%H:%M:%S' 2>/dev/null || echo "+$((retry_at - now))s")"
echo " Push a fix (any new commit clears this), or: rm $FAIL_FILE"
exit 0
fi
print_info "Backoff elapsed — retrying $MIRROR (failure #$((FAIL_COUNT + 1)) if this fails)"
# A queue written under the pre-architecture name belongs to x86_64.
if [[ "$arch" == "x86_64" && ! -f "$state_file" && -f "$(legacy_sync_queue_file "$MIRROR")" ]]; then
state_file=$(legacy_sync_queue_file "$MIRROR")
fi
if [[ "$arch" == "x86_64" && ! -f "$fail_file" && -f "$(legacy_sync_fail_file "$MIRROR")" ]]; then
fail_file=$(legacy_sync_fail_file "$MIRROR")
fi
fi
# Non-blocking: a build in progress means this tick has nothing to do.
if ! try_release_lock; then
holder=$(release_lock_holder)
print_info "A release is already running (${holder:-holder unknown}) — skipping this tick"
exit 0
fi
# Nothing queued: stay quiet. At a 5-minute cadence this is most
# invocations, and a header for each would bury the runs that matter in
# the journal.
[[ -f "$state_file" ]] || return 0
print_info "State file found: $STATE_FILE"
print_info "Starting release workflow for $MIRROR..."
print_header "Processing Sync for $MIRROR ($arch)"
if "$BUILD_ROOT/bin/repo" release --mirror "$MIRROR" --skip-prod-check; then
print_success "Release completed successfully for $MIRROR"
rm -f "$STATE_FILE"
rm -f "$FAIL_FILE"
print_success "State file removed: $STATE_FILE"
else
status=$?
FAIL_COUNT=$((FAIL_COUNT + 1))
cat >"$FAIL_FILE" <<EOF
FAILURE_COUNT=$FAIL_COUNT
local fail_count=0 failed_at failed_fingerprint delay now retry_at
if [[ -f "$fail_file" ]]; then
FAILURE_COUNT=0 FAILURE_AT=0 FAILURE_FINGERPRINT=""
# shellcheck disable=SC1090
source "$fail_file" 2>/dev/null || true
fail_count="${FAILURE_COUNT:-0}"
failed_at="${FAILURE_AT:-0}"
failed_fingerprint="${FAILURE_FINGERPRINT:-}"
if [[ "$failed_fingerprint" != "$(current_fingerprint)" ]]; then
print_info "Repository changed since the last failure — clearing backoff and retrying"
rm -f "$fail_file"
fail_count=0
else
delay=$(backoff_seconds "$fail_count")
now=$(date +%s)
retry_at=$((failed_at + delay))
if ((now < retry_at)); then
print_warning "$MIRROR ($arch) has failed $fail_count time(s) on this tree — not retrying until $(date -d "@$retry_at" '+%H:%M:%S' 2>/dev/null || echo "+$((retry_at - now))s")"
echo " Push a fix (any new commit clears this), or: rm $fail_file"
return 0
fi
print_info "Backoff elapsed — retrying $MIRROR ($arch) (failure #$((fail_count + 1)) if this fails)"
fi
fi
# Non-blocking: a build in progress means this tick has nothing to do. The
# lock is reentrant, so once this run holds it the remaining architectures
# run under the same acquisition.
if ! try_release_lock; then
local holder
holder=$(release_lock_holder)
print_info "A release is already running (${holder:-holder unknown}) — skipping this tick"
return 0
fi
print_info "State file found: $state_file"
print_info "Starting release workflow for $MIRROR ($arch)..."
if "$BUILD_ROOT/bin/repo" release --mirror "$MIRROR" --arch "$arch" --skip-prod-check; then
print_success "Release completed successfully for $MIRROR ($arch)"
rm -f "$state_file" "$fail_file"
print_success "State file removed: $state_file"
return 0
fi
fail_count=$((fail_count + 1))
cat >"$fail_file" <<EOF
FAILURE_COUNT=$fail_count
FAILURE_AT=$(date +%s)
FAILURE_FINGERPRINT=$(current_fingerprint)
EOF
next=$(backoff_seconds "$FAIL_COUNT")
print_error "Release failed for $MIRROR (attempt $FAIL_COUNT)"
print_warning "State file retained for retry: $STATE_FILE"
local next
next=$(backoff_seconds "$fail_count")
print_error "Release failed for $MIRROR ($arch) (attempt $fail_count)"
print_warning "State file retained for retry: $state_file"
print_warning "Backing off $((next / 60))m before the next attempt; a new commit retries sooner"
exit "$status"
fi
return 1
}
status=0
for arch in $ARCHES; do
release_arch "$arch" || status=1
done
exit "$status"
+9 -5
View File
@@ -164,11 +164,15 @@ mkdir -p "$BUILD_OUTPUT_DIR" "$REPO_DIR" "$SRC_DIR"
# Check Docker is available
check_docker
# Setup QEMU for aarch64 builds on x86_64 hosts
if [[ "$(uname -m)" == "x86_64" && "$ARCH" == "aarch64" ]]; then
# Check if QEMU is already working
if ! docker run --rm --platform linux/arm64 alpine:3.21 /bin/true >/dev/null 2>&1; then
print_info "Setting up QEMU for ARM64 emulation..."
# A foreign target architecture runs under QEMU user emulation. Probe by
# actually running a container for the target platform: that is the only
# test that covers both "binfmt not registered" and "registered but broken".
HOST_ARCH=$(uname -m)
[[ "$HOST_ARCH" == "arm64" ]] && HOST_ARCH=aarch64
if [[ "$HOST_ARCH" != "$ARCH" ]]; then
# shellcheck disable=SC2086
if ! docker run --rm $(get_platform_arg "$ARCH") alpine:3.21 /bin/true >/dev/null 2>&1; then
print_info "Setting up QEMU for $ARCH emulation on this $HOST_ARCH host..."
setup_qemu
fi
fi
+37 -11
View File
@@ -1,6 +1,10 @@
#!/bin/bash
# Check PKGBUILD versions against published repo versions
# Creates state files for edge and/or stable if any packages need building
# Creates a state file per channel and architecture when packages need building
#
# Usage: check-versions [--pull] [--arch <arch>]
# --pull pull the repository first (the scheduled run does this)
# --arch <arch> check one architecture; default: every published one
set -e
@@ -10,11 +14,27 @@ source "$BUILD_ROOT/helpers/paths.sh"
source "$BUILD_ROOT/helpers/package-metadata.sh"
source "$BUILD_ROOT/helpers/lock-helpers.sh"
STATE_DIR="${OMARCHY_STATE_DIR:-/root/.state}"
ARCH="${ARCH:-x86_64}"
PULL=false
ARCHES=""
[[ "${1:-}" == "--pull" ]] && PULL=true
while [[ $# -gt 0 ]]; do
case $1 in
--pull)
PULL=true
shift
;;
--arch)
require_valid_arch "$2"
ARCHES="$2"
shift 2
;;
*)
print_error "Unknown option: $1"
exit 1
;;
esac
done
[[ -n "$ARCHES" ]] || ARCHES=$(published_arches)
mkdir -p "$STATE_DIR"
@@ -147,11 +167,12 @@ check_package() {
check_mirror() {
local mirror="$1"
local state_file="$STATE_DIR/.sync-needed-$mirror"
local state_file
state_file=$(sync_queue_file "$mirror" "$ARCH")
local needs_build=false
local packages=()
print_info "Checking $mirror packages..."
print_info "Checking $mirror packages for $ARCH..."
while IFS= read -r pkg; do
local pkgdir="$PKGBUILDS_DIR/$pkg"
@@ -168,18 +189,23 @@ check_mirror() {
# this to name the packages in its start report, which is the difference
# between "a build is running" and "your package is in this build".
printf '%s\n' "${packages[@]}" >"$state_file"
print_success "${mirror^} needs building (${#packages[@]} packages)"
print_success "${mirror^} ($ARCH) needs building (${#packages[@]} packages)"
print_info "Packages: ${packages[*]}"
print_info "State file created: $state_file"
else
print_info "${mirror^} is up to date"
print_info "${mirror^} ($ARCH) is up to date"
fi
echo ""
}
check_mirror edge
check_mirror rc
check_mirror stable
# One pass per published architecture: the version comparison reads that
# architecture's channel databases, and each queue is its own file.
for ARCH in $ARCHES; do
update_arch_paths
check_mirror edge
check_mirror rc
check_mirror stable
done
print_success "Version check complete!"
+4 -1
View File
@@ -33,7 +33,10 @@ clean_packages() {
# Extract package name (remove version and architecture)
# Format: name-version-release-arch.pkg.tar.*
local pkgname=$(echo "$pkg" | sed -E 's/-[0-9]+.*-(any|x86_64|i686)\.pkg\.tar\..*//')
# The version starts at the first "-<digit>" and neither pkgver nor pkgrel
# can contain a hyphen, so the architecture is whatever sits between the
# last hyphen and .pkg.tar — any, x86_64, aarch64, and whatever comes next.
local pkgname=$(echo "$pkg" | sed -E 's/-[0-9]+.*-[^-]+\.pkg\.tar\..*//')
# Add to array
if [[ -n "${packages[$pkgname]}" ]]; then
+18 -6
View File
@@ -28,7 +28,13 @@ ISO_REPO="${OMARCHY_ISO_REPO:-omacom-io/omarchy-iso}"
DEV_BRANCH="${OMARCHY_DEV_BRANCH:-quattro}"
PKGS_DB_BASE="${OMARCHY_PKGS_DB_BASE:-https://pkgs.omarchy.org}"
RC_DB_URL="$PKGS_DB_BASE/rc/x86_64/omarchy.db.tar.zst"
# The train observes channels through the reference architecture (the first
# published one). The pair is pinned to one version for every architecture and
# built for all of them in one rc trigger, so one architecture's database is
# the state of the release; per-architecture drift is a build failure the
# scheduled pipeline reports, not something the train has to poll for.
OBSERVED_ARCH=$(reference_arch)
RC_DB_URL="$PKGS_DB_BASE/rc/$OBSERVED_ARCH/omarchy.db.tar.zst"
SRCDEST_DIR="$BUILD_ROOT/.srcdest"
MIRROR_CLONE="$SRCDEST_DIR/omarchy" # bare mirror (shared with bin/omarchy-pkgs)
@@ -186,7 +192,7 @@ published_version() {
# A unique query string busts the CDN cache: right after a sync the plain
# URL can keep serving the previous db for a while, which reads as "not
# published yet" to status, the wait loop, and ship's pre-checks.
if ! curl -sf "$PKGS_DB_BASE/$channel/x86_64/omarchy.db.tar.zst?$(date +%s%N)" -o "$tmp"; then
if ! curl -sf "$PKGS_DB_BASE/$channel/$OBSERVED_ARCH/omarchy.db.tar.zst?$(date +%s%N)" -o "$tmp"; then
rm -f "$tmp"
return 2
fi
@@ -251,8 +257,12 @@ fi
git -C /root/omarchy-pkgs-rc fetch origin rc
git -C /root/omarchy-pkgs-rc reset --hard origin/rc
cd /root/omarchy-pkgs-rc
OMARCHY_RC_PINS=1 OMARCHY_REPO_ROOT=/root/omarchy-pkgs/pkgs.omarchy.org \
bin/repo release --mirror rc --package omarchy omarchy-settings --skip-prod-check
# The pair is built once per published architecture (helpers/paths.sh on the
# host decides which), so every architecture'"'"'s rc channel carries the pins.
for arch in $(BUILD_ROOT=/root/omarchy-pkgs-rc bash -c "source helpers/paths.sh; published_arches"); do
OMARCHY_RC_PINS=1 OMARCHY_REPO_ROOT=/root/omarchy-pkgs/pkgs.omarchy.org \
bin/repo release --mirror rc --arch "$arch" --package omarchy omarchy-settings --skip-prod-check
done
'
trigger_rc_build() {
@@ -278,10 +288,12 @@ host_advance() { # host_advance <from> <to> [extra args...]
# against this machine's (likely stale) local tree would be wrong.
if ! resolve_repo_host "$REPO_HOST_OVERRIDE" >/dev/null && ! on_repo_host; then
print_no_host_help "advance $from -> $to" \
" cd /root/omarchy-pkgs && bin/repo advance --from $from --to $to --skip-prod-check $*"
" cd /root/omarchy-pkgs && bin/repo advance --from $from --to $to --arch all --skip-prod-check $*"
return 1
fi
"$BUILD_ROOT/bin/repo" advance --from "$from" --to "$to" --skip-prod-check "$@"
# --arch all: the host advances every architecture it publishes, so a train
# never moves a channel for one architecture and not another.
"$BUILD_ROOT/bin/repo" advance --from "$from" --to "$to" --arch all --skip-prod-check "$@"
}
wait_for_published() { # wait_for_published <channel> <version> [timeout-seconds]
+5 -1
View File
@@ -114,7 +114,11 @@ BUILT_FILES=""
# What the scheduled version check queued, when it was the one that asked for
# this run. Absent for a manual run, which is fine — the report just omits it.
QUEUED_PACKAGES=""
QUEUE_FILE="${OMARCHY_STATE_DIR:-/root/.state}/.sync-needed-$MIRROR"
QUEUE_FILE=$(sync_queue_file "$MIRROR" "$ARCH")
# A queue written under the pre-architecture name belongs to x86_64.
if [[ "$ARCH" == "x86_64" && ! -s "$QUEUE_FILE" && -s "$(legacy_sync_queue_file "$MIRROR")" ]]; then
QUEUE_FILE=$(legacy_sync_queue_file "$MIRROR")
fi
[[ -s "$QUEUE_FILE" ]] && QUEUED_PACKAGES=$(grep -c '' "$QUEUE_FILE")
if [[ "$DRY_RUN" != true ]]; then
+4
View File
@@ -237,6 +237,10 @@ echo ""
# --- release timers ----------------------------------------------------------
print_info "Published architectures: $(published_arches | tr '\n' ' ')"
echo " (PUBLISHED_ARCHES in helpers/paths.sh, or OMARCHY_ARCHES in $CREDENTIALS)"
echo ""
TIMERS=(omarchy-check-versions omarchy-auto-release-edge omarchy-auto-release-rc omarchy-auto-release-stable)
if [[ "$SKIP_TIMERS" == true ]]; then
+16 -10
View File
@@ -84,15 +84,18 @@ echo ""
# --- queued work -------------------------------------------------------------
print_info "Queued builds (state files in $STATE_DIR)"
print_info "Queued builds (state files in $STATE_DIR; architectures: $(published_arches | tr '\n' ' '))"
queued=false
for channel in edge rc stable; do
state_file="$STATE_DIR/.sync-needed-$channel"
if [[ -f "$state_file" ]]; then
queued=true
since=$(date -r "$state_file" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "unknown")
printf ' • %-7s queued since %s\n' "$channel" "$since"
fi
for arch in $(published_arches); do
state_file=$(sync_queue_file "$channel" "$arch")
[[ "$arch" == "x86_64" && ! -f "$state_file" ]] && state_file=$(legacy_sync_queue_file "$channel")
if [[ -f "$state_file" ]]; then
queued=true
since=$(date -r "$state_file" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "unknown")
printf ' • %-7s %-8s queued since %s\n' "$channel" "$arch" "$since"
fi
done
done
[[ "$queued" == false ]] && echo " (nothing queued — all channels up to date)"
echo ""
@@ -101,7 +104,9 @@ echo ""
# say so plainly: it is queued but deliberately not being retried yet.
paused=false
for channel in edge rc stable; do
fail_file="$STATE_DIR/.build-failed-$channel"
for arch in $(published_arches); do
fail_file=$(sync_fail_file "$channel" "$arch")
[[ "$arch" == "x86_64" && ! -f "$fail_file" ]] && fail_file=$(legacy_sync_fail_file "$channel")
[[ -f "$fail_file" ]] || continue
if [[ "$paused" == false ]]; then
print_error "Failing builds (backoff active)"
@@ -122,13 +127,14 @@ for channel in edge rc stable; do
else
when="retries on the next tick"
fi
printf ' ✗ %-7s %s consecutive failure(s), %s\n' "$channel" "$FAILURE_COUNT" "$when"
printf ' ✗ %-7s %-8s %s consecutive failure(s), %s\n' "$channel" "$arch" "$FAILURE_COUNT" "$when"
printf ' last attempt %s on commit %s\n' \
"$(date -d "@$FAILURE_AT" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "$FAILURE_AT")" \
"${FAILURE_FINGERPRINT:0:12}"
done
done
if [[ "$paused" == true ]]; then
echo " Any new commit clears the backoff; or: rm $STATE_DIR/.build-failed-<channel>"
echo " Any new commit clears the backoff; or: rm $STATE_DIR/.build-failed-<channel>-<arch>"
echo ""
fi