diff --git a/bin/advance-channel b/bin/advance-channel new file mode 100755 index 0000000..4fc7411 --- /dev/null +++ b/bin/advance-channel @@ -0,0 +1,299 @@ +#!/bin/bash +# Move packages forward through the channel pipeline: edge -> rc -> stable. +# +# Copies package artifacts AND their detached signatures from one channel to +# the next, driven by the source channel's database (not the raw directory, so +# historical files never leak forward), then cleans, rebuilds, and syncs the +# destination. Published filenames are never overwritten: a same-name file +# that already exists at the destination is skipped (and reported when its +# bytes differ), because the R2 cache cannot recover from a rewrite. + +set -e + +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +source "$BUILD_ROOT/helpers/message-helpers.sh" +source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/lock-helpers.sh" +source "$BUILD_ROOT/helpers/package-metadata.sh" + +FROM="" +TO="" +DRY_RUN=false +SYNC_REMOTE="" +SKIP_PROD_CHECK=false +FAST_RING_ONLY=false +BOOTSTRAP=false +PACKAGES="" + +usage() { + echo "Usage: $0 --from --to [OPTIONS]" + echo "" + echo "Directions:" + echo " --from edge --to rc Open a release train: carry edge forward into rc" + echo " --from rc --to stable Ship: promote the tested rc channel to stable" + echo " --from stable --to rc Only with --fast-ring (parity replication)" + echo " or --bootstrap (one-time initial seed)" + echo "" + echo "Options:" + echo " --arch Target architecture (x86_64 or aarch64, default: x86_64)" + echo " --package Advance only the named package(s)" + echo " --fast-ring Restrict to fast-ring packages (stable -> rc parity copy)" + echo " --bootstrap One-time stable -> rc seed before forward-only enforcement" + echo " --dry-run Preview without changing files" + echo " --sync-remote Rclone remote for sync (default: pkgs.omarchy.org:omarchy-pkgs)" + echo " --skip-prod-check Skip production confirmation during sync" + echo " -h, --help Show this help message" + echo "" + echo "What it does:" + echo " 1) Read the source channel database for the current package set" + echo " 2) Copy eligible packages + .sig files not yet at the destination" + echo " 3) Clean the destination (keep 2 versions)" + echo " 4) Rebuild the destination database" + echo " 5) Sync the destination to the remote (packages first, database last)" + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --from) + FROM="$2" + shift 2 + ;; + --to) + TO="$2" + shift 2 + ;; + --arch) + ARCH="$2" + update_arch_paths + shift 2 + ;; + --package) + shift + while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do + PACKAGES="$PACKAGES $1" + shift + done + PACKAGES="${PACKAGES# }" + ;; + --fast-ring) + FAST_RING_ONLY=true + shift + ;; + --bootstrap) + BOOTSTRAP=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --sync-remote) + SYNC_REMOTE="$2" + shift 2 + ;; + --skip-prod-check) + SKIP_PROD_CHECK=true + shift + ;; + -h | --help) + usage 0 + ;; + *) + print_error "Unknown option: $1" + usage 1 + ;; + esac +done + +require_valid_mirror "$FROM" +require_valid_mirror "$TO" + +# The pipeline only moves forward. stable -> rc is allowed for exactly two +# labeled purposes: fast-ring parity replication and the one-time bootstrap. +# edge -> stable is the legacy migrate path, kept for the transition cycle. +case "$FROM->$TO" in +"edge->rc" | "rc->stable") ;; +"edge->stable") + print_warning "edge -> stable is the legacy migrate path; new releases flow edge -> rc -> stable" + ;; +"stable->rc") + if [[ "$FAST_RING_ONLY" != true && "$BOOTSTRAP" != true ]]; then + print_error "stable -> rc requires --fast-ring (parity replication) or --bootstrap (initial seed)" + exit 1 + fi + ;; +*) + print_error "Refusing $FROM -> $TO: packages only move forward (edge -> rc -> stable)" + exit 1 + ;; +esac + +SOURCE_DIR="$REPO_ROOT/$FROM/$ARCH" +TARGET_DIR="$REPO_ROOT/$TO/$ARCH" +SOURCE_DB="$SOURCE_DIR/omarchy.db.tar.zst" + +print_header "Advance Channel: $FROM -> $TO" +print_info "Architecture: $ARCH" +print_info "Source: $SOURCE_DIR" +print_info "Target: $TARGET_DIR" +[[ "$FAST_RING_ONLY" == true ]] && print_info "Scope: fast-ring packages only" +[[ "$BOOTSTRAP" == true ]] && print_info "Mode: bootstrap (initial rc seed)" +[[ -n "$PACKAGES" ]] && print_info "Packages: $PACKAGES" + +if [[ ! -f "$SOURCE_DB" ]]; then + print_error "Source database not found: $SOURCE_DB" + echo "Nothing has been published to the $FROM channel on this host." + exit 1 +fi + +if [[ "$DRY_RUN" == true ]]; then + print_warning "DRY RUN MODE - No changes will be made" +else + acquire_release_lock || exit 1 +fi + +# Manifest: "namebasefilename" per current package in the source db. +# Each desc record begins with %FILENAME%, so that marker both closes the +# previous record and opens the next. +read_manifest() { + tar -xOf "$SOURCE_DB" --wildcards '*/desc' 2>/dev/null | awk ' + function emit() { + if (name != "" && filename != "") printf "%s\t%s\t%s\n", name, base, filename + name = ""; base = ""; filename = "" + } + $0 == "%FILENAME%" { emit(); getline; filename = $0; next } + $0 == "%NAME%" { getline; name = $0; next } + $0 == "%BASE%" { getline; base = $0; next } + END { emit() } + ' +} + +package_wanted() { + local name="$1" base="$2" + [[ -z "$PACKAGES" ]] && return 0 + local want + for want in $PACKAGES; do + [[ "$want" == "$name" || "$want" == "$base" ]] && return 0 + done + return 1 +} + +# Split packages publish under their pkgname; eligibility metadata lives in +# the pkgbase directory. Packages whose PKGBUILD has since been removed from +# this repo default to moving: they are part of the source channel's set and +# leaving them behind would hole the destination. +package_eligible() { + local name="$1" base="$2" + local pkgdir + + pkgdir=$(package_dir_for_name "$name" 2>/dev/null) || + pkgdir=$(package_dir_for_name "$base" 2>/dev/null) || pkgdir="" + + if [[ -z "$pkgdir" ]]; then + [[ "$FAST_RING_ONLY" == true ]] && return 1 + return 0 + fi + + if [[ "$FAST_RING_ONLY" == true ]]; then + package_is_fast_ring "$pkgdir" || return 1 + fi + + package_moves_to_channel "$pkgdir" "$TO" +} + +mkdir -p "$TARGET_DIR" + +COPIED=0 +PRESENT=0 +SKIPPED=0 +MISSING_FILES=() +MISSING_SIGS=() +DIFFERING=() + +while IFS=$'\t' read -r name base filename; do + package_wanted "$name" "$base" || continue + if ! package_eligible "$name" "$base"; then + SKIPPED=$((SKIPPED + 1)) + continue + fi + + src="$SOURCE_DIR/$filename" + sig="$src.sig" + dest="$TARGET_DIR/$filename" + + if [[ ! -f "$src" ]]; then + MISSING_FILES+=("$filename") + continue + fi + if [[ ! -f "$sig" ]]; then + MISSING_SIGS+=("$filename") + continue + fi + + if [[ -f "$dest" ]]; then + if cmp -s "$src" "$dest"; then + PRESENT=$((PRESENT + 1)) + else + DIFFERING+=("$filename") + fi + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + echo " would copy: $filename (+ .sig)" + else + cp -p "$src" "$dest" + cp -p "$sig" "$dest.sig" + echo " copied: $filename (+ .sig)" + fi + COPIED=$((COPIED + 1)) +done < <(read_manifest) + +echo "" +print_info "Copied: $COPIED | Already present: $PRESENT | Not eligible: $SKIPPED" + +if [[ ${#MISSING_FILES[@]} -gt 0 ]]; then + print_error "${#MISSING_FILES[@]} package(s) named in the $FROM database are missing on disk:" + printf ' %s\n' "${MISSING_FILES[@]}" + echo "The $FROM channel is inconsistent; rebuild its database before advancing." + exit 1 +fi + +if [[ ${#MISSING_SIGS[@]} -gt 0 ]]; then + print_error "${#MISSING_SIGS[@]} package(s) have no detached signature in $FROM:" + printf ' %s\n' "${MISSING_SIGS[@]}" + echo "Signatures must travel with their packages. Re-sign these in $FROM" + echo "(bin/repo sign) and re-run." + exit 1 +fi + +if [[ ${#DIFFERING[@]} -gt 0 ]]; then + print_warning "${#DIFFERING[@]} file(s) already published in $TO with different bytes (kept as-is):" + printf ' %s\n' "${DIFFERING[@]}" + echo "Published filenames are never rewritten. The destination copy stays" + echo "authoritative; a genuinely new build needs a new pkgver/pkgrel." +fi + +if [[ "$DRY_RUN" == true ]]; then + print_info "Dry run: skipping clean, database update, and sync" + exit 0 +fi + +print_info "Cleaning $TO repository..." +"$BUILD_ROOT/bin/clean-repo" --mirror "$TO" --arch "$ARCH" + +print_info "Updating $TO repository database..." +"$BUILD_ROOT/bin/update-repo" --mirror "$TO" --arch "$ARCH" + +echo "" +print_info "Syncing $TO repository to remote..." +SYNC_ARGS=("--mirror" "$TO" "--arch" "$ARCH") +[[ -n "$SYNC_REMOTE" ]] && SYNC_ARGS+=("--remote" "$SYNC_REMOTE") +[[ "$SKIP_PROD_CHECK" == true ]] && SYNC_ARGS+=("--skip-prod-check") +"$BUILD_ROOT/bin/sync-repo" "${SYNC_ARGS[@]}" || { + print_error "Sync failed" + exit 1 +} + +print_success "Advance complete: $FROM -> $TO" diff --git a/bin/migrate-edge-to-stable b/bin/migrate-edge-to-stable deleted file mode 100755 index 64d6acd..0000000 --- a/bin/migrate-edge-to-stable +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash -# Promote tested edge package artifacts into stable without overwriting - -set -e - -BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") -source "$BUILD_ROOT/helpers/message-helpers.sh" -source "$BUILD_ROOT/helpers/paths.sh" - -DRY_RUN=false -SYNC_REMOTE="" -SKIP_PROD_CHECK=false -PACKAGE="" - -while [[ $# -gt 0 ]]; do - case $1 in - --arch) - ARCH="$2" - update_arch_paths - shift 2 - ;; - --package) - PACKAGE="$2" - shift 2 - ;; - --dry-run) - DRY_RUN=true - shift - ;; - --sync-remote) - SYNC_REMOTE="$2" - shift 2 - ;; - --skip-prod-check) - SKIP_PROD_CHECK=true - shift - ;; - -h | --help) - echo "Usage: $0 [OPTIONS]" - echo "" - echo "Options:" - echo " --arch Target architecture (x86_64 or aarch64, default: x86_64)" - echo " --package Migrate only the specified package" - echo " --dry-run Preview migration and cleanup without changing files" - echo " --sync-remote Rclone remote for sync (default: pkgs.omarchy.org:omarchy-pkgs)" - echo " --skip-prod-check Skip production environment check during sync" - echo " -h, --help Show this help message" - echo "" - echo "What it does:" - echo " 1) Rsync tested package artifacts from edge -> stable (without overwriting existing files)" - echo " 2) Run stable cleanup (keep 2 versions)" - echo " 3) Update stable repository database" - echo " 4) Sync stable repository to remote (prompts for confirmation)" - exit 0 - ;; - *) - print_error "Unknown option: $1" - exit 1 - ;; - esac -done - -SOURCE_DIR="$BUILD_ROOT/pkgs.omarchy.org/edge/$ARCH" -TARGET_DIR="$BUILD_ROOT/pkgs.omarchy.org/stable/$ARCH" - -print_header "Promote Edge Artifacts to Stable" -print_info "Architecture: $ARCH" -print_info "Source: $SOURCE_DIR" -print_info "Target: $TARGET_DIR" - -if [[ ! -d "$SOURCE_DIR" ]]; then - print_error "Source directory not found: $SOURCE_DIR" - exit 1 -fi - -# Version segment starts with a digit (same assumption as bin/clean-repo), -# so omarchy-nvim doesn't also match omarchy-nvim-extras -PACKAGE_GLOB="*" -if [[ -n "$PACKAGE" ]]; then - print_info "Package: $PACKAGE" - PACKAGE_GLOB="$PACKAGE-[0-9]*" - if ! compgen -G "$SOURCE_DIR/$PACKAGE_GLOB.pkg.tar.*" >/dev/null; then - print_error "No artifacts matching $PACKAGE_GLOB.pkg.tar.* found in $SOURCE_DIR" - exit 1 - fi -fi - -mkdir -p "$TARGET_DIR" - -if [[ "$DRY_RUN" == true ]]; then - print_warning "DRY RUN MODE - No changes will be made" -fi - -print_info "Syncing package files from edge to stable (ignore existing)..." -RSYNC_FLAGS="-av --ignore-existing --include=*/ --include=$PACKAGE_GLOB.pkg.tar.* --exclude=*" -if [[ "$DRY_RUN" == true ]]; then - RSYNC_FLAGS="$RSYNC_FLAGS -n" -fi - -# shellcheck disable=SC2086 -rsync $RSYNC_FLAGS "$SOURCE_DIR/" "$TARGET_DIR/" - -print_info "Cleaning stable repository..." -if [[ "$DRY_RUN" == true ]]; then - "$BUILD_ROOT/bin/repo" clean --mirror stable --arch "$ARCH" --dry-run - print_info "Skipping database update in dry run mode" -else - "$BUILD_ROOT/bin/repo" clean --mirror stable --arch "$ARCH" - print_info "Updating stable repository database..." - "$BUILD_ROOT/bin/repo" update --mirror stable --arch "$ARCH" -fi - -if [[ "$DRY_RUN" != true ]]; then - echo "" - print_info "Syncing stable repository to remote..." - SYNC_ARGS=("--mirror" "stable" "--arch" "$ARCH") - if [[ -n "$SYNC_REMOTE" ]]; then - SYNC_ARGS+=("--remote" "$SYNC_REMOTE") - fi - if [[ "$SKIP_PROD_CHECK" == true ]]; then - SYNC_ARGS+=("--skip-prod-check") - fi - "$BUILD_ROOT/bin/sync-repo" "${SYNC_ARGS[@]}" || { - print_error "Sync failed" - exit 1 - } -fi - -print_success "Migration workflow complete" diff --git a/bin/release b/bin/release index 0f92eb7..6e6d41e 100755 --- a/bin/release +++ b/bin/release @@ -6,6 +6,7 @@ set -e BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") source "$BUILD_ROOT/helpers/message-helpers.sh" source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/lock-helpers.sh" source "$BUILD_ROOT/helpers/basecamp-notifier.sh" SYNC_REMOTE="" @@ -85,6 +86,12 @@ print_info "Target architecture: $ARCH" print_info "Mirror: $MIRROR" print_info "Build workspace: $BUILD_OUTPUT_DIR" +# One channel mutation at a time: a timer firing mid-run or a second operator +# waits here instead of interleaving a partial publish. +if [[ "$DRY_RUN" != true ]]; then + acquire_release_lock || exit 1 +fi + # Step 1: Build echo "" if [[ "$DRY_RUN" == true ]]; then @@ -156,5 +163,25 @@ fi exit 1 } +# Step 7 (stable only): fast-ring packages publish to rc and stable together, +# so rc never falls behind what stable users actually run. Skipped until the +# rc channel has been bootstrapped. +if [[ "$MIRROR" == "stable" ]]; then + echo "" + if [[ -f "$REPO_ROOT/rc/$ARCH/omarchy.db.tar.zst" ]]; then + print_info "Step 7: Replicating fast-ring packages to rc (parity)..." + REPLICATE_ARGS=(--from stable --to rc --fast-ring --arch "$ARCH") + [[ -n "$SYNC_REMOTE" ]] && REPLICATE_ARGS+=(--sync-remote "$SYNC_REMOTE") + [[ "$SKIP_PROD_CHECK" == true ]] && REPLICATE_ARGS+=(--skip-prod-check) + "$BUILD_ROOT/bin/advance-channel" "${REPLICATE_ARGS[@]}" || { + print_error "Fast-ring replication to rc failed" + notify_error "Release failed: rc parity replication failed" "Mirror: $MIRROR | Arch: $ARCH" + exit 1 + } + else + print_info "rc channel not bootstrapped; skipping fast-ring replication" + fi +fi + echo "" print_success "Release workflow completed successfully!" diff --git a/bin/repo b/bin/repo index 48c91cc..b195447 100755 --- a/bin/repo +++ b/bin/repo @@ -54,7 +54,9 @@ show_usage() { echo " promote Promote build to production (build-output → pkgs.omarchy.org)" echo " update Update repository database" echo " clean Clean old package versions" - echo " migrate Migrate edge packages into stable (no overwrite), then clean + update" + echo " advance Move packages forward through the pipeline (edge → rc → stable)" + echo " bootstrap-rc One-time seed of the rc channel from stable" + echo " migrate DEPRECATED alias for 'advance --from edge --to stable'" echo " list List source package metadata (use --repo for published repo)" echo " remove Remove a specific package" echo " sync Sync repository to remote" @@ -112,8 +114,17 @@ clean) "$SCRIPT_DIR/clean-repo" "$@" 2>&1 | tee "$LOG_FILE" exit ${PIPESTATUS[0]} ;; +advance) + "$SCRIPT_DIR/advance-channel" "$@" 2>&1 | tee "$LOG_FILE" + exit ${PIPESTATUS[0]} + ;; +bootstrap-rc) + "$SCRIPT_DIR/advance-channel" --from stable --to rc --bootstrap "$@" 2>&1 | tee "$LOG_FILE" + exit ${PIPESTATUS[0]} + ;; migrate) - "$SCRIPT_DIR/migrate-edge-to-stable" "$@" 2>&1 | tee "$LOG_FILE" + print_warning "'migrate' is deprecated; releases now flow edge → rc → stable (bin/repo advance)" + "$SCRIPT_DIR/advance-channel" --from edge --to stable "$@" 2>&1 | tee "$LOG_FILE" exit ${PIPESTATUS[0]} ;; list) diff --git a/bin/upload-prebuilt b/bin/upload-prebuilt index d4b4831..5a798b9 100755 --- a/bin/upload-prebuilt +++ b/bin/upload-prebuilt @@ -4,6 +4,11 @@ set -e SCRIPT_DIR=$(realpath "${BASH_SOURCE[0]%/*}") +BUILD_ROOT=$(realpath "$SCRIPT_DIR/..") +source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/lock-helpers.sh" + +acquire_release_lock || exit 1 # Only sync understands the publishing flags; sign, promote and update reject # unknown options outright, so they cannot be forwarded blindly. diff --git a/helpers/lock-helpers.sh b/helpers/lock-helpers.sh new file mode 100644 index 0000000..01a110e --- /dev/null +++ b/helpers/lock-helpers.sh @@ -0,0 +1,44 @@ +# Release lock: one channel mutation at a time on this host. +# +# Everything that changes a published channel (release runs, advance, promote, +# upload-prebuilt) takes this lock, so a timer firing mid-advance or two +# operators colliding serializes instead of interleaving partial publishes. +# +# The lock lives beside the published tree (REPO_ROOT), not the checkout, so +# the primary checkout and the rc branch worktree contend on the same file. +# Reentrant across child scripts: acquire_release_lock exports +# OMARCHY_RELEASE_LOCK_HELD, and children skip acquisition when they see it +# (the flock fd is inherited, so the lock stays held for the whole tree). + +RELEASE_LOCK_FD=9 + +acquire_release_lock() { + local timeout="${1:-3600}" + + if [[ -n "${OMARCHY_RELEASE_LOCK_HELD:-}" ]]; then + return 0 + fi + + local lock_file="${REPO_ROOT:-$BUILD_ROOT/pkgs.omarchy.org}/.release.lock" + mkdir -p "$(dirname "$lock_file")" + + eval "exec $RELEASE_LOCK_FD>>\"\$lock_file\"" + + if ! flock -n "$RELEASE_LOCK_FD"; then + local holder + holder=$(cat "$lock_file" 2>/dev/null | tail -1) + echo "Waiting for release lock (up to ${timeout}s)${holder:+ — held by: $holder}" >&2 + if ! flock -w "$timeout" "$RELEASE_LOCK_FD"; then + echo "Could not acquire release lock within ${timeout}s: $lock_file" >&2 + echo "If no release is actually running, remove the file and retry." >&2 + return 1 + fi + fi + + # Record the holder for the "waiting for" message above. Truncate first so a + # crashed holder's stale line does not linger once we own the lock. + : >"$lock_file" + echo "pid $$ ($0) since $(date '+%Y-%m-%d %H:%M:%S')" >>"$lock_file" + + export OMARCHY_RELEASE_LOCK_HELD=1 +}