Publish completed packages when a peer build fails

2026-09-14 02:30:11 -04:00
1 parent 3626590e94
commit d96b950901
@@ -44,4 +44,5 @@ jobs:
              ./bin/sync-rebuilds --self-test
              ./bin/omarchy-pkgs self-test
              ./bin/omarchy-release self-test
+              ./tests/partial-release.sh
            '
@@ -125,6 +125,14 @@ bin/repo advance --from edge --to rc

 The release command is smart and **incremental** - it only builds packages that have changed or are missing. You generally don't need to specify a package manually unless you are debugging a specific failure.

+When a package fails, a completed build run still signs and publishes the packages
+that succeeded. Failed packages and their blocked dependents remain queued with
+failure backoff; retries compare against the updated repository and skip the
+published versions. Only artifacts recorded by fully completed package builds
+are eligible for a partial release. An interrupted build, a failed publication
+step, or an incomplete pair using deferred runtime dependencies still stops the
+release. Reports distinguish partial publication from complete success.
+
 ```bash
 # Build changed/new packages, sign, promote, clean, update, and sync
 bin/repo release
@@ -320,6 +320,20 @@ echo "  Skipped:        ${#SKIPPED_PACKAGES[@]} (up-to-date or excluded)"
 echo "  Failed:         ${#FAILED_PACKAGES[@]}"
 echo "  Blocked:        ${#BLOCKED_PACKAGES[@]}"

+# The release caller supplies a fresh directory. A completed result
+# distinguishes package failures from an interrupted/failed orchestrator;
+# only artifacts belonging to fully successful builds may be published.
+if [[ -n "${OMARCHY_BUILD_RESULT_DIR:-}" ]]; then
+  mkdir -p "$OMARCHY_BUILD_RESULT_DIR"
+  : > "$OMARCHY_BUILD_RESULT_DIR/artifacts"
+  for package in "${SUCCESSFUL_PACKAGES[@]}"; do
+    cat "$PLAN_DIR/artifacts/$package" >> "$OMARCHY_BUILD_RESULT_DIR/artifacts"
+  done
+  printf '%s\n' "${FAILED_PACKAGES[@]}" | sed '/^$/d' > "$OMARCHY_BUILD_RESULT_DIR/failed"
+  printf '%s\n' "${BLOCKED_PACKAGES[@]}" | sed '/^$/d' > "$OMARCHY_BUILD_RESULT_DIR/blocked"
+  touch "$OMARCHY_BUILD_RESULT_DIR/complete"
+fi
+
 if (( ${#FAILED_PACKAGES[@]} + ${#BLOCKED_PACKAGES[@]} )); then
  if (( ${#FAILED_PACKAGES[@]} )); then
    echo "Failed packages:"
@@ -330,7 +344,9 @@ if (( ${#FAILED_PACKAGES[@]} + ${#BLOCKED_PACKAGES[@]} )); then
    printf '  - %s\n' "${BLOCKED_PACKAGES[@]}"
  fi
  print_warning "Some packages failed (see details above)"
-  exit 1
+  # Reserved for a completed run with unsuccessful packages. Other failures
+  # must not let release publish arbitrary files left in the workspace.
+  exit 2
 fi

 print_success "Build completed successfully!"
@@ -138,11 +138,43 @@ if [[ "$DRY_RUN" == true ]]; then
 else
  print_info "Step 1/6: Building packages..."
 fi
-"$BUILD_ROOT/bin/build" "${BUILD_ARGS[@]}" || {
+BUILD_RESULT_DIR=$(mktemp -d)
+trap 'rm -rf "$BUILD_RESULT_DIR"' EXIT
+build_status=0
+OMARCHY_BUILD_RESULT_DIR="$BUILD_RESULT_DIR" "$BUILD_ROOT/bin/build" "${BUILD_ARGS[@]}" || build_status=$?
+partial=false
+if [[ "$build_status" == 2 && -f "$BUILD_RESULT_DIR/complete" && "$DRY_RUN" != true ]]; then
+  if [[ "${OMARCHY_DEFER_RUNTIME_DEPS:-false}" == true ]]; then
+    print_error "The deferred release pair must succeed together; nothing will be published"
+    notify_error "Release failed: Incomplete release pair" "$RELEASE_CONTEXT"
+    exit 1
+  fi
+  partial=true
+  while IFS= read -r file; do
+    [[ -f "$BUILD_OUTPUT_DIR/$file" ]] || {
+      print_error "Completed build artifact is missing: $file"
+      notify_error "Release failed: Missing completed artifact" "$RELEASE_CONTEXT"
+      exit 1
+    }
+  done < "$BUILD_RESULT_DIR/artifacts"
+  # Exclude partial split outputs or leftovers from failed builds. Moving
+  # them out of the flat publication directory keeps them available for
+  # diagnosis without handing them to sign/promote.
+  unpublished=""
+  for path in "$BUILD_OUTPUT_DIR"/*.pkg.tar.*; do
+    [[ -f "$path" ]] || continue
+    file=${path##*/}
+    if ! grep -Fxq -- "${file%.sig}" "$BUILD_RESULT_DIR/artifacts"; then
+      [[ -n "$unpublished" ]] || unpublished=$(mktemp -d "$BUILD_OUTPUT_DIR/.unpublished.XXXXXX")
+      mv -- "$path" "$unpublished/"
+    fi
+  done
+  print_warning "Some packages failed; publishing the completed packages before retrying the failures"
+elif [[ "$build_status" != 0 ]]; then
  print_error "Build failed"
  notify_error "Release failed: Build step failed" "$RELEASE_CONTEXT"
  exit 1
-}
+fi

 if [[ "$DRY_RUN" == true ]]; then
  echo ""
@@ -155,6 +187,12 @@ BUILT_COUNT=$(grep -c '' <<<"$BUILT_FILES")
 [[ -z "$BUILT_FILES" ]] && BUILT_COUNT=0
 print_info "Built $BUILT_COUNT package(s) this run"

+if [[ "$partial" == true && "$BUILT_COUNT" == 0 ]]; then
+  print_error "No completed packages to publish"
+  notify_error "Release failed: No completed packages" "$RELEASE_CONTEXT"
+  exit 1
+fi
+
 # Step 2: Sign
 echo ""
 print_info "Step 2/6: Signing packages..."
@@ -213,7 +251,16 @@ if ((BUILT_COUNT > 0)); then
  summary+="<br><br><strong>$BUILT_COUNT package(s) published:</strong>"
  summary+="$(format_package_list_html "$BUILT_FILES")"
  summary+="<br><br>Live at https://pkgs.omarchy.org/$MIRROR/$ARCH/"
-  notify_success "Release published: $MIRROR" "$summary"
+  if [[ "$partial" == true ]]; then
+    failed=$(tr '\n' ' ' < "$BUILD_RESULT_DIR/failed" | basecamp_html_escape)
+    blocked=$(tr '\n' ' ' < "$BUILD_RESULT_DIR/blocked" | basecamp_html_escape)
+    [[ -z "$failed" ]] || summary+="<br><br>Failed: $failed"
+    [[ -z "$blocked" ]] || summary+="<br>Blocked by failed dependencies: $blocked"
+    summary+="<br>Published packages will be skipped on retry; failed packages remain queued."
+    notify_info "Partial release published: $MIRROR" "$summary"
+  else
+    notify_success "Release published: $MIRROR" "$summary"
+  fi
 else
  # Rare by construction: a release only runs when the version check queued
  # work, so publishing nothing means the check and the builder disagreed
@@ -228,4 +275,10 @@ else
 fi

 echo ""
+if [[ "$partial" == true ]]; then
+  print_warning "Completed packages published; unsuccessful packages remain for retry"
+  # Preserve the scheduled queue and failure backoff. The next version
+  # check/build compares against the updated repository and skips successes.
+  exit 1
+fi
 print_success "Release workflow completed successfully!"
@@ -411,6 +411,11 @@ build_package() {
      ln -sf omarchy-build.db.tar.zst omarchy-build.db || return 1
    fi

+    # A release may publish successful builds even when a peer fails. Record
+    # outputs only after this package's entire split build has completed.
+    mkdir -p "$BUILD_PLAN_DIR/artifacts" || return 1
+    printf '%s\n' "${new_pkgs[@]}" > "$BUILD_PLAN_DIR/artifacts/$pkg" || return 1
+
    echo "    Successfully built $pkg"
    return 0
  else
@@ -89,11 +89,17 @@ EOF
 fixture consumer "checkdepends_${TEST_ARCH}=('broken')"
 fixture transitive "makedepends=('consumer')"
 fixture independent ''
-if run_build transitive consumer broken independent > "$TEST_ROOT/failure.log" 2>&1; then
+failure_status=0
+OMARCHY_BUILD_RESULT_DIR="$TEST_ROOT/results" run_build transitive consumer broken independent > "$TEST_ROOT/failure.log" 2>&1 || failure_status=$?
+if [[ "$failure_status" != 2 ]]; then
  cat "$TEST_ROOT/failure.log"
-  echo 'FAIL: a failed prerequisite did not fail the run' >&2
+  echo "FAIL: expected completed package failure (2), got $failure_status" >&2
  exit 1
 fi
+[[ -f "$TEST_ROOT/results/complete" ]]
+[[ $(cat "$TEST_ROOT/results/failed") == broken ]]
+[[ $(cat "$TEST_ROOT/results/blocked") == $'transitive\nconsumer' || $(cat "$TEST_ROOT/results/blocked") == $'consumer\ntransitive' ]]
+[[ $(cat "$TEST_ROOT/results/artifacts") == "independent-1-1-$TEST_ARCH.pkg.tar.zst" ]]
 grep -q 'consumer blocked by unsuccessful dependency: broken' "$TEST_ROOT/failure.log"
 grep -q 'transitive blocked by unsuccessful dependency: consumer' "$TEST_ROOT/failure.log"
 compgen -G "$TEST_ROOT/build-output/edge/$TEST_ARCH/independent-1-1-*.pkg.tar.zst" >/dev/null
@@ -0,0 +1,154 @@
+#!/bin/bash
+# Exercise release/queue handling without publishing to an external repository.
+set -euo pipefail
+BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
+TEST_ROOT=$(mktemp -d)
+trap 'rm -rf "$TEST_ROOT"' EXIT
+export TEST_ROOT
+export OMARCHY_STATE_DIR="$TEST_ROOT/state"
+unset OMARCHY_REPO_ROOT OMARCHY_KEEP_BUILD_WORKSPACE OMARCHY_DEFER_RUNTIME_DEPS
+mkdir -p "$TEST_ROOT/bin" "$TEST_ROOT/state"
+cp "$BUILD_ROOT/bin/"{release,auto-release,check-versions} "$TEST_ROOT/bin/"
+cp -r "$BUILD_ROOT/helpers" "$BUILD_ROOT/build" "$TEST_ROOT/"
+cat >> "$TEST_ROOT/helpers/basecamp-notifier.sh" <<'EOF'
+notify_basecamp() { printf '%s\n' "$1" >> "$TEST_ROOT/notifications"; }
+EOF
+
+cat > "$TEST_ROOT/bin/build" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+[[ "$MODE" == plan ]] && exit 0
+out="$TEST_ROOT/build-output/edge/x86_64"
+mkdir -p "$out"
+echo complete > "$out/good-1-1-x86_64.pkg.tar.zst"
+[[ "$MODE" == infrastructure ]] && exit 1
+[[ "$MODE" == interrupted ]] && exit 2
+mkdir -p "$OMARCHY_BUILD_RESULT_DIR"
+: > "$OMARCHY_BUILD_RESULT_DIR/artifacts"
+: > "$OMARCHY_BUILD_RESULT_DIR/failed"
+: > "$OMARCHY_BUILD_RESULT_DIR/blocked"
+if [[ "$MODE" != all_failed ]]; then
+  echo good-1-1-x86_64.pkg.tar.zst > "$OMARCHY_BUILD_RESULT_DIR/artifacts"
+fi
+touch "$OMARCHY_BUILD_RESULT_DIR/complete"
+[[ "$MODE" == success ]] && exit 0
+[[ "$MODE" == missing_artifact ]] && echo missing-1-1-x86_64.pkg.tar.zst >> "$OMARCHY_BUILD_RESULT_DIR/artifacts"
+# A failed split build may have copied its first output before failing.
+echo incomplete > "$out/bad-first-1-1-x86_64.pkg.tar.zst"
+echo bad > "$OMARCHY_BUILD_RESULT_DIR/failed"
+echo blocked > "$OMARCHY_BUILD_RESULT_DIR/blocked"
+exit 2
+EOF
+cat > "$TEST_ROOT/bin/repo" <<'EOF'
+#!/bin/bash
+shift
+exec "$(dirname "$0")/release" "$@"
+EOF
+cat > "$TEST_ROOT/bin/stage" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+stage=$(basename "$0")
+echo "$stage" >> "$TEST_ROOT/stages"
+out="$TEST_ROOT/build-output/edge/x86_64"
+repo="$TEST_ROOT/pkgs.omarchy.org/edge/x86_64"
+case "$stage" in
+  sign)
+    [[ "$MODE" == signing ]] && exit 1
+    for file in "$out"/*.pkg.tar.zst; do
+      echo "signed $(basename "$file")" >> "$TEST_ROOT/signed"
+      touch "$file.sig"
+    done
+    ;;
+  promote-build)
+    mkdir -p "$repo"
+    mv "$out"/*.pkg.tar.zst* "$repo/"
+    ;;
+  update-repo)
+    mkdir -p "$TEST_ROOT/db/good-1-1"
+    printf '%%NAME%%\ngood\n\n%%BASE%%\ngood\n\n%%VERSION%%\n1-1\n' > "$TEST_ROOT/db/good-1-1/desc"
+    tar --zstd -cf "$repo/omarchy.db.tar.zst" -C "$TEST_ROOT/db" good-1-1
+    ln -s omarchy.db.tar.zst "$repo/omarchy.db"
+    ;;
+esac
+EOF
+for stage in sign promote-build clean-repo update-repo sync-repo; do
+  cp "$TEST_ROOT/bin/stage" "$TEST_ROOT/bin/$stage"
+done
+chmod +x "$TEST_ROOT/bin/"*
+
+for package in good bad blocked; do
+  mkdir -p "$TEST_ROOT/pkgbuilds/$package/.omarchy"
+  printf '{"source":"local"}\n' > "$TEST_ROOT/pkgbuilds/$package/.omarchy/package.json"
+  printf "pkgname=%s\npkgver=1\npkgrel=1\narch=('x86_64')\n" "$package" > "$TEST_ROOT/pkgbuilds/$package/PKGBUILD"
+done
+echo "depends=('bad')" >> "$TEST_ROOT/pkgbuilds/blocked/PKGBUILD"
+
+reset_case() {
+  rm -rf "$TEST_ROOT/build-output" "$TEST_ROOT/pkgs.omarchy.org" "$TEST_ROOT/state" "$TEST_ROOT/db"
+  mkdir -p "$TEST_ROOT/state"
+  : > "$TEST_ROOT/stages"
+  : > "$TEST_ROOT/signed"
+  : > "$TEST_ROOT/notifications"
+  printf 'good\nbad\nblocked\n' > "$TEST_ROOT/state/.sync-needed-edge-x86_64"
+}
+
+reset_case
+export MODE=partial
+if "$TEST_ROOT/bin/auto-release" edge x86_64 > "$TEST_ROOT/partial.log" 2>&1; then
+  echo 'FAIL: partial release cleared failure status' >&2; exit 1
+fi
+[[ $(cat "$TEST_ROOT/stages") == $'sign\npromote-build\nclean-repo\nupdate-repo\nsync-repo' ]]
+[[ $(cat "$TEST_ROOT/signed") == 'signed good-1-1-x86_64.pkg.tar.zst' ]]
+[[ -f "$TEST_ROOT/pkgs.omarchy.org/edge/x86_64/good-1-1-x86_64.pkg.tar.zst" ]]
+[[ ! -e "$TEST_ROOT/pkgs.omarchy.org/edge/x86_64/bad-first-1-1-x86_64.pkg.tar.zst" ]]
+[[ -f "$TEST_ROOT/state/.build-failed-edge-x86_64" ]]
+grep -q 'Partial release published: edge' "$TEST_ROOT/notifications"
+grep -q 'Failed: bad' "$TEST_ROOT/notifications"
+grep -q 'Blocked by failed dependencies: blocked' "$TEST_ROOT/notifications"
+"$TEST_ROOT/bin/check-versions" > "$TEST_ROOT/versions.log" 2>&1
+[[ $(cat "$TEST_ROOT/state/.sync-needed-edge-x86_64") == $'bad\nblocked' ]]
+ARCH=x86_64 MIRROR=edge DRY_RUN=true PKGBUILDS_DIR="$TEST_ROOT/pkgbuilds" \
+  FINAL_OUTPUT_DIR="$TEST_ROOT/pkgs.omarchy.org/edge/x86_64" HELPERS_DIR="$TEST_ROOT/helpers" \
+  "$TEST_ROOT/build/build.sh" > "$TEST_ROOT/retry.log" 2>&1
+grep -q 'good - already up to date' "$TEST_ROOT/retry.log"
+grep -q 'Build order: bad blocked' "$TEST_ROOT/retry.log"
+echo 'PASS: successes publish; incomplete outputs stay out; retries skip published versions'
+
+for MODE in infrastructure interrupted all_failed missing_artifact; do
+  export MODE
+  reset_case
+  if "$TEST_ROOT/bin/release" --mirror edge > "$TEST_ROOT/$MODE.log" 2>&1; then
+    echo "FAIL: $MODE succeeded" >&2; exit 1
+  fi
+  [[ ! -s "$TEST_ROOT/stages" ]]
+done
+echo 'PASS: infrastructure failures, missing completion records, and zero successes never publish'
+
+reset_case
+export MODE=signing
+if "$TEST_ROOT/bin/release" --mirror edge > "$TEST_ROOT/signing.log" 2>&1; then
+  echo 'FAIL: signing failure succeeded' >&2; exit 1
+fi
+[[ $(cat "$TEST_ROOT/stages") == sign ]]
+echo 'PASS: signing failure stops promotion and sync'
+
+reset_case
+export MODE=partial
+if OMARCHY_DEFER_RUNTIME_DEPS=true "$TEST_ROOT/bin/release" --mirror edge > "$TEST_ROOT/pair.log" 2>&1; then
+  echo 'FAIL: incomplete deferred release pair published' >&2; exit 1
+fi
+[[ ! -s "$TEST_ROOT/stages" ]]
+echo 'PASS: deferred release pairs cannot publish partially'
+
+reset_case
+export MODE=success
+"$TEST_ROOT/bin/auto-release" edge x86_64 > "$TEST_ROOT/success.log" 2>&1
+[[ ! -e "$TEST_ROOT/state/.sync-needed-edge-x86_64" ]]
+grep -q 'Release published: edge' "$TEST_ROOT/notifications"
+echo 'PASS: complete success publishes and clears the queue'
+
+reset_case
+export MODE=plan
+"$TEST_ROOT/bin/release" --mirror edge --dry-run > "$TEST_ROOT/plan.log" 2>&1
+[[ ! -s "$TEST_ROOT/stages" && ! -s "$TEST_ROOT/notifications" ]]
+echo 'PASS: dry runs neither publish nor notify'