diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d0a68e..a1a307a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,4 +44,5 @@ jobs: ./bin/sync-rebuilds --self-test ./bin/omarchy-pkgs self-test ./bin/omarchy-release self-test + ./tests/partial-release.sh ' diff --git a/README.md b/README.md index 64a5d58..2ba3ce5 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,14 @@ bin/repo advance --from edge --to rc The release command is smart and **incremental** - it only builds packages that have changed or are missing. You generally don't need to specify a package manually unless you are debugging a specific failure. +When a package fails, a completed build run still signs and publishes the packages +that succeeded. Failed packages and their blocked dependents remain queued with +failure backoff; retries compare against the updated repository and skip the +published versions. Only artifacts recorded by fully completed package builds +are eligible for a partial release. An interrupted build, a failed publication +step, or an incomplete pair using deferred runtime dependencies still stops the +release. Reports distinguish partial publication from complete success. + ```bash # Build changed/new packages, sign, promote, clean, update, and sync bin/repo release diff --git a/bin/build b/bin/build index 0b16b59..5c8eba8 100755 --- a/bin/build +++ b/bin/build @@ -320,6 +320,20 @@ echo " Skipped: ${#SKIPPED_PACKAGES[@]} (up-to-date or excluded)" echo " Failed: ${#FAILED_PACKAGES[@]}" echo " Blocked: ${#BLOCKED_PACKAGES[@]}" +# The release caller supplies a fresh directory. A completed result +# distinguishes package failures from an interrupted/failed orchestrator; +# only artifacts belonging to fully successful builds may be published. +if [[ -n "${OMARCHY_BUILD_RESULT_DIR:-}" ]]; then + mkdir -p "$OMARCHY_BUILD_RESULT_DIR" + : > "$OMARCHY_BUILD_RESULT_DIR/artifacts" + for package in "${SUCCESSFUL_PACKAGES[@]}"; do + cat "$PLAN_DIR/artifacts/$package" >> "$OMARCHY_BUILD_RESULT_DIR/artifacts" + done + printf '%s\n' "${FAILED_PACKAGES[@]}" | sed '/^$/d' > "$OMARCHY_BUILD_RESULT_DIR/failed" + printf '%s\n' "${BLOCKED_PACKAGES[@]}" | sed '/^$/d' > "$OMARCHY_BUILD_RESULT_DIR/blocked" + touch "$OMARCHY_BUILD_RESULT_DIR/complete" +fi + if (( ${#FAILED_PACKAGES[@]} + ${#BLOCKED_PACKAGES[@]} )); then if (( ${#FAILED_PACKAGES[@]} )); then echo "Failed packages:" @@ -330,7 +344,9 @@ if (( ${#FAILED_PACKAGES[@]} + ${#BLOCKED_PACKAGES[@]} )); then printf ' - %s\n' "${BLOCKED_PACKAGES[@]}" fi print_warning "Some packages failed (see details above)" - exit 1 + # Reserved for a completed run with unsuccessful packages. Other failures + # must not let release publish arbitrary files left in the workspace. + exit 2 fi print_success "Build completed successfully!" diff --git a/bin/release b/bin/release index 193bfb5..4a1d704 100755 --- a/bin/release +++ b/bin/release @@ -138,11 +138,43 @@ if [[ "$DRY_RUN" == true ]]; then else print_info "Step 1/6: Building packages..." fi -"$BUILD_ROOT/bin/build" "${BUILD_ARGS[@]}" || { +BUILD_RESULT_DIR=$(mktemp -d) +trap 'rm -rf "$BUILD_RESULT_DIR"' EXIT +build_status=0 +OMARCHY_BUILD_RESULT_DIR="$BUILD_RESULT_DIR" "$BUILD_ROOT/bin/build" "${BUILD_ARGS[@]}" || build_status=$? +partial=false +if [[ "$build_status" == 2 && -f "$BUILD_RESULT_DIR/complete" && "$DRY_RUN" != true ]]; then + if [[ "${OMARCHY_DEFER_RUNTIME_DEPS:-false}" == true ]]; then + print_error "The deferred release pair must succeed together; nothing will be published" + notify_error "Release failed: Incomplete release pair" "$RELEASE_CONTEXT" + exit 1 + fi + partial=true + while IFS= read -r file; do + [[ -f "$BUILD_OUTPUT_DIR/$file" ]] || { + print_error "Completed build artifact is missing: $file" + notify_error "Release failed: Missing completed artifact" "$RELEASE_CONTEXT" + exit 1 + } + done < "$BUILD_RESULT_DIR/artifacts" + # Exclude partial split outputs or leftovers from failed builds. Moving + # them out of the flat publication directory keeps them available for + # diagnosis without handing them to sign/promote. + unpublished="" + for path in "$BUILD_OUTPUT_DIR"/*.pkg.tar.*; do + [[ -f "$path" ]] || continue + file=${path##*/} + if ! grep -Fxq -- "${file%.sig}" "$BUILD_RESULT_DIR/artifacts"; then + [[ -n "$unpublished" ]] || unpublished=$(mktemp -d "$BUILD_OUTPUT_DIR/.unpublished.XXXXXX") + mv -- "$path" "$unpublished/" + fi + done + print_warning "Some packages failed; publishing the completed packages before retrying the failures" +elif [[ "$build_status" != 0 ]]; then print_error "Build failed" notify_error "Release failed: Build step failed" "$RELEASE_CONTEXT" exit 1 -} +fi if [[ "$DRY_RUN" == true ]]; then echo "" @@ -155,6 +187,12 @@ BUILT_COUNT=$(grep -c '' <<<"$BUILT_FILES") [[ -z "$BUILT_FILES" ]] && BUILT_COUNT=0 print_info "Built $BUILT_COUNT package(s) this run" +if [[ "$partial" == true && "$BUILT_COUNT" == 0 ]]; then + print_error "No completed packages to publish" + notify_error "Release failed: No completed packages" "$RELEASE_CONTEXT" + exit 1 +fi + # Step 2: Sign echo "" print_info "Step 2/6: Signing packages..." @@ -213,7 +251,16 @@ if ((BUILT_COUNT > 0)); then summary+="

$BUILT_COUNT package(s) published:" summary+="$(format_package_list_html "$BUILT_FILES")" summary+="

Live at https://pkgs.omarchy.org/$MIRROR/$ARCH/" - notify_success "Release published: $MIRROR" "$summary" + if [[ "$partial" == true ]]; then + failed=$(tr '\n' ' ' < "$BUILD_RESULT_DIR/failed" | basecamp_html_escape) + blocked=$(tr '\n' ' ' < "$BUILD_RESULT_DIR/blocked" | basecamp_html_escape) + [[ -z "$failed" ]] || summary+="

Failed: $failed" + [[ -z "$blocked" ]] || summary+="
Blocked by failed dependencies: $blocked" + summary+="
Published packages will be skipped on retry; failed packages remain queued." + notify_info "Partial release published: $MIRROR" "$summary" + else + notify_success "Release published: $MIRROR" "$summary" + fi else # Rare by construction: a release only runs when the version check queued # work, so publishing nothing means the check and the builder disagreed @@ -228,4 +275,10 @@ else fi echo "" +if [[ "$partial" == true ]]; then + print_warning "Completed packages published; unsuccessful packages remain for retry" + # Preserve the scheduled queue and failure backoff. The next version + # check/build compares against the updated repository and skips successes. + exit 1 +fi print_success "Release workflow completed successfully!" diff --git a/build/build.sh b/build/build.sh index bcf02a4..96500e7 100755 --- a/build/build.sh +++ b/build/build.sh @@ -411,6 +411,11 @@ build_package() { ln -sf omarchy-build.db.tar.zst omarchy-build.db || return 1 fi + # A release may publish successful builds even when a peer fails. Record + # outputs only after this package's entire split build has completed. + mkdir -p "$BUILD_PLAN_DIR/artifacts" || return 1 + printf '%s\n' "${new_pkgs[@]}" > "$BUILD_PLAN_DIR/artifacts/$pkg" || return 1 + echo " Successfully built $pkg" return 0 else diff --git a/tests/build-isolation.sh b/tests/build-isolation.sh index 27c9df5..7f1250f 100755 --- a/tests/build-isolation.sh +++ b/tests/build-isolation.sh @@ -89,11 +89,17 @@ EOF fixture consumer "checkdepends_${TEST_ARCH}=('broken')" fixture transitive "makedepends=('consumer')" fixture independent '' -if run_build transitive consumer broken independent > "$TEST_ROOT/failure.log" 2>&1; then +failure_status=0 +OMARCHY_BUILD_RESULT_DIR="$TEST_ROOT/results" run_build transitive consumer broken independent > "$TEST_ROOT/failure.log" 2>&1 || failure_status=$? +if [[ "$failure_status" != 2 ]]; then cat "$TEST_ROOT/failure.log" - echo 'FAIL: a failed prerequisite did not fail the run' >&2 + echo "FAIL: expected completed package failure (2), got $failure_status" >&2 exit 1 fi +[[ -f "$TEST_ROOT/results/complete" ]] +[[ $(cat "$TEST_ROOT/results/failed") == broken ]] +[[ $(cat "$TEST_ROOT/results/blocked") == $'transitive\nconsumer' || $(cat "$TEST_ROOT/results/blocked") == $'consumer\ntransitive' ]] +[[ $(cat "$TEST_ROOT/results/artifacts") == "independent-1-1-$TEST_ARCH.pkg.tar.zst" ]] grep -q 'consumer blocked by unsuccessful dependency: broken' "$TEST_ROOT/failure.log" grep -q 'transitive blocked by unsuccessful dependency: consumer' "$TEST_ROOT/failure.log" compgen -G "$TEST_ROOT/build-output/edge/$TEST_ARCH/independent-1-1-*.pkg.tar.zst" >/dev/null diff --git a/tests/partial-release.sh b/tests/partial-release.sh new file mode 100755 index 0000000..c4fc725 --- /dev/null +++ b/tests/partial-release.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Exercise release/queue handling without publishing to an external repository. +set -euo pipefail +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +TEST_ROOT=$(mktemp -d) +trap 'rm -rf "$TEST_ROOT"' EXIT +export TEST_ROOT +export OMARCHY_STATE_DIR="$TEST_ROOT/state" +unset OMARCHY_REPO_ROOT OMARCHY_KEEP_BUILD_WORKSPACE OMARCHY_DEFER_RUNTIME_DEPS +mkdir -p "$TEST_ROOT/bin" "$TEST_ROOT/state" +cp "$BUILD_ROOT/bin/"{release,auto-release,check-versions} "$TEST_ROOT/bin/" +cp -r "$BUILD_ROOT/helpers" "$BUILD_ROOT/build" "$TEST_ROOT/" +cat >> "$TEST_ROOT/helpers/basecamp-notifier.sh" <<'EOF' +notify_basecamp() { printf '%s\n' "$1" >> "$TEST_ROOT/notifications"; } +EOF + +cat > "$TEST_ROOT/bin/build" <<'EOF' +#!/bin/bash +set -euo pipefail +[[ "$MODE" == plan ]] && exit 0 +out="$TEST_ROOT/build-output/edge/x86_64" +mkdir -p "$out" +echo complete > "$out/good-1-1-x86_64.pkg.tar.zst" +[[ "$MODE" == infrastructure ]] && exit 1 +[[ "$MODE" == interrupted ]] && exit 2 +mkdir -p "$OMARCHY_BUILD_RESULT_DIR" +: > "$OMARCHY_BUILD_RESULT_DIR/artifacts" +: > "$OMARCHY_BUILD_RESULT_DIR/failed" +: > "$OMARCHY_BUILD_RESULT_DIR/blocked" +if [[ "$MODE" != all_failed ]]; then + echo good-1-1-x86_64.pkg.tar.zst > "$OMARCHY_BUILD_RESULT_DIR/artifacts" +fi +touch "$OMARCHY_BUILD_RESULT_DIR/complete" +[[ "$MODE" == success ]] && exit 0 +[[ "$MODE" == missing_artifact ]] && echo missing-1-1-x86_64.pkg.tar.zst >> "$OMARCHY_BUILD_RESULT_DIR/artifacts" +# A failed split build may have copied its first output before failing. +echo incomplete > "$out/bad-first-1-1-x86_64.pkg.tar.zst" +echo bad > "$OMARCHY_BUILD_RESULT_DIR/failed" +echo blocked > "$OMARCHY_BUILD_RESULT_DIR/blocked" +exit 2 +EOF +cat > "$TEST_ROOT/bin/repo" <<'EOF' +#!/bin/bash +shift +exec "$(dirname "$0")/release" "$@" +EOF +cat > "$TEST_ROOT/bin/stage" <<'EOF' +#!/bin/bash +set -euo pipefail +stage=$(basename "$0") +echo "$stage" >> "$TEST_ROOT/stages" +out="$TEST_ROOT/build-output/edge/x86_64" +repo="$TEST_ROOT/pkgs.omarchy.org/edge/x86_64" +case "$stage" in + sign) + [[ "$MODE" == signing ]] && exit 1 + for file in "$out"/*.pkg.tar.zst; do + echo "signed $(basename "$file")" >> "$TEST_ROOT/signed" + touch "$file.sig" + done + ;; + promote-build) + mkdir -p "$repo" + mv "$out"/*.pkg.tar.zst* "$repo/" + ;; + update-repo) + mkdir -p "$TEST_ROOT/db/good-1-1" + printf '%%NAME%%\ngood\n\n%%BASE%%\ngood\n\n%%VERSION%%\n1-1\n' > "$TEST_ROOT/db/good-1-1/desc" + tar --zstd -cf "$repo/omarchy.db.tar.zst" -C "$TEST_ROOT/db" good-1-1 + ln -s omarchy.db.tar.zst "$repo/omarchy.db" + ;; +esac +EOF +for stage in sign promote-build clean-repo update-repo sync-repo; do + cp "$TEST_ROOT/bin/stage" "$TEST_ROOT/bin/$stage" +done +chmod +x "$TEST_ROOT/bin/"* + +for package in good bad blocked; do + mkdir -p "$TEST_ROOT/pkgbuilds/$package/.omarchy" + printf '{"source":"local"}\n' > "$TEST_ROOT/pkgbuilds/$package/.omarchy/package.json" + printf "pkgname=%s\npkgver=1\npkgrel=1\narch=('x86_64')\n" "$package" > "$TEST_ROOT/pkgbuilds/$package/PKGBUILD" +done +echo "depends=('bad')" >> "$TEST_ROOT/pkgbuilds/blocked/PKGBUILD" + +reset_case() { + rm -rf "$TEST_ROOT/build-output" "$TEST_ROOT/pkgs.omarchy.org" "$TEST_ROOT/state" "$TEST_ROOT/db" + mkdir -p "$TEST_ROOT/state" + : > "$TEST_ROOT/stages" + : > "$TEST_ROOT/signed" + : > "$TEST_ROOT/notifications" + printf 'good\nbad\nblocked\n' > "$TEST_ROOT/state/.sync-needed-edge-x86_64" +} + +reset_case +export MODE=partial +if "$TEST_ROOT/bin/auto-release" edge x86_64 > "$TEST_ROOT/partial.log" 2>&1; then + echo 'FAIL: partial release cleared failure status' >&2; exit 1 +fi +[[ $(cat "$TEST_ROOT/stages") == $'sign\npromote-build\nclean-repo\nupdate-repo\nsync-repo' ]] +[[ $(cat "$TEST_ROOT/signed") == 'signed good-1-1-x86_64.pkg.tar.zst' ]] +[[ -f "$TEST_ROOT/pkgs.omarchy.org/edge/x86_64/good-1-1-x86_64.pkg.tar.zst" ]] +[[ ! -e "$TEST_ROOT/pkgs.omarchy.org/edge/x86_64/bad-first-1-1-x86_64.pkg.tar.zst" ]] +[[ -f "$TEST_ROOT/state/.build-failed-edge-x86_64" ]] +grep -q 'Partial release published: edge' "$TEST_ROOT/notifications" +grep -q 'Failed: bad' "$TEST_ROOT/notifications" +grep -q 'Blocked by failed dependencies: blocked' "$TEST_ROOT/notifications" +"$TEST_ROOT/bin/check-versions" > "$TEST_ROOT/versions.log" 2>&1 +[[ $(cat "$TEST_ROOT/state/.sync-needed-edge-x86_64") == $'bad\nblocked' ]] +ARCH=x86_64 MIRROR=edge DRY_RUN=true PKGBUILDS_DIR="$TEST_ROOT/pkgbuilds" \ + FINAL_OUTPUT_DIR="$TEST_ROOT/pkgs.omarchy.org/edge/x86_64" HELPERS_DIR="$TEST_ROOT/helpers" \ + "$TEST_ROOT/build/build.sh" > "$TEST_ROOT/retry.log" 2>&1 +grep -q 'good - already up to date' "$TEST_ROOT/retry.log" +grep -q 'Build order: bad blocked' "$TEST_ROOT/retry.log" +echo 'PASS: successes publish; incomplete outputs stay out; retries skip published versions' + +for MODE in infrastructure interrupted all_failed missing_artifact; do + export MODE + reset_case + if "$TEST_ROOT/bin/release" --mirror edge > "$TEST_ROOT/$MODE.log" 2>&1; then + echo "FAIL: $MODE succeeded" >&2; exit 1 + fi + [[ ! -s "$TEST_ROOT/stages" ]] +done +echo 'PASS: infrastructure failures, missing completion records, and zero successes never publish' + +reset_case +export MODE=signing +if "$TEST_ROOT/bin/release" --mirror edge > "$TEST_ROOT/signing.log" 2>&1; then + echo 'FAIL: signing failure succeeded' >&2; exit 1 +fi +[[ $(cat "$TEST_ROOT/stages") == sign ]] +echo 'PASS: signing failure stops promotion and sync' + +reset_case +export MODE=partial +if OMARCHY_DEFER_RUNTIME_DEPS=true "$TEST_ROOT/bin/release" --mirror edge > "$TEST_ROOT/pair.log" 2>&1; then + echo 'FAIL: incomplete deferred release pair published' >&2; exit 1 +fi +[[ ! -s "$TEST_ROOT/stages" ]] +echo 'PASS: deferred release pairs cannot publish partially' + +reset_case +export MODE=success +"$TEST_ROOT/bin/auto-release" edge x86_64 > "$TEST_ROOT/success.log" 2>&1 +[[ ! -e "$TEST_ROOT/state/.sync-needed-edge-x86_64" ]] +grep -q 'Release published: edge' "$TEST_ROOT/notifications" +echo 'PASS: complete success publishes and clears the queue' + +reset_case +export MODE=plan +"$TEST_ROOT/bin/release" --mirror edge --dry-run > "$TEST_ROOT/plan.log" 2>&1 +[[ ! -s "$TEST_ROOT/stages" && ! -s "$TEST_ROOT/notifications" ]] +echo 'PASS: dry runs neither publish nor notify'