Fix nine defects in the push/sync path found in review
The worst was fatal: push passed --skip-prod-check to upload-prebuilt, which
forwards every argument to sign, promote and update as well, and sign rejects
unknown options. Every non-dry-run push and deploy would have uploaded and
verified its artifacts and then failed before signing. upload-prebuilt now
routes publishing flags to sync alone.
The partial-tree guard was weaker than it looked:
- it counted archive files locally against package names in the remote
database, and this tree keeps two versions per package, so a checkout with
a spare version of half the repository could pass while still hiding
hundreds of packages. It now compares package-name sets and lists what
would be hidden.
- it treated any unreadable remote as an empty one, so an auth failure or a
corrupt database disabled it. Only rclone's "directory not found" now
counts as a fresh mirror; every other failure aborts.
Also:
- sync had no set -e, so a failed package upload fell through to publishing
the database, advertising packages that were never uploaded. Each transfer
is now checked before the next step.
- --package with no names silently meant "every package", which under --yes
could publish everything from one unset variable in a script.
- push now refuses to run when the host has packages staged from an earlier
failure, since publishing would sign and promote those too.
- epoch versions contain a colon, which rsync reads as host:path, so no
package with an epoch could be transferred. Sources are ./-prefixed.
- remote paths are quoted for the remote shell.
- sync spun forever on a missing option value.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
David Heinemeier HanssonandClaude Opus 5 committed
1 parent
c5f5f1c12c
commit
fd9c078bf2
5 files changed
+192
-33
No files matched your search
+59
-7
@@ -17,8 +17,10 @@ HOST=""
|
||||
REMOTE_ROOT="/root/omarchy-pkgs"
|
||||
CREDENTIALS="/root/.omarchy/build-credentials"
|
||||
PACKAGES=""
|
||||
PACKAGE_FLAG_GIVEN=false
|
||||
DRY_RUN=false
|
||||
ASSUME_YES=false
|
||||
INCLUDE_STAGED=false
|
||||
|
||||
print_header "Push Build to Host"
|
||||
|
||||
@@ -40,9 +42,9 @@ while [[ $# -gt 0 ]]; do
|
||||
;;
|
||||
--package)
|
||||
shift
|
||||
PACKAGES=""
|
||||
PACKAGE_FLAG_GIVEN=true
|
||||
while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do
|
||||
PACKAGES="$PACKAGES $1"
|
||||
[[ -n "$1" ]] && PACKAGES="$PACKAGES $1"
|
||||
shift
|
||||
done
|
||||
PACKAGES="${PACKAGES# }"
|
||||
@@ -63,6 +65,10 @@ while [[ $# -gt 0 ]]; do
|
||||
ASSUME_YES=true
|
||||
shift
|
||||
;;
|
||||
--include-staged)
|
||||
INCLUDE_STAGED=true
|
||||
shift
|
||||
;;
|
||||
-h | --help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo ""
|
||||
@@ -77,6 +83,7 @@ while [[ $# -gt 0 ]]; do
|
||||
echo " --remote-root <path> Repository path on the host (default: $REMOTE_ROOT)"
|
||||
echo " --dry-run Show what would be pushed, transfer nothing"
|
||||
echo " -y, --yes Do not ask for confirmation"
|
||||
echo " --include-staged Publish packages already staged on the host too"
|
||||
echo " -h, --help Show this help message"
|
||||
echo ""
|
||||
echo "Typical use:"
|
||||
@@ -122,6 +129,14 @@ fi
|
||||
# is rebuilt there, so neither should ride along.
|
||||
mapfile -t ALL_FILES < <(cd "$BUILD_OUTPUT_DIR" && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig$' || true)
|
||||
|
||||
# "--package" with nothing after it, or with an empty variable, must not quietly
|
||||
# widen to every artifact — that is the difference between shipping one package
|
||||
# and shipping whatever else happens to be lying around.
|
||||
if [[ "$PACKAGE_FLAG_GIVEN" == true && -z "$PACKAGES" ]]; then
|
||||
print_error "--package requires at least one package name"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILES=()
|
||||
if [[ -z "$PACKAGES" ]]; then
|
||||
FILES=("${ALL_FILES[@]}")
|
||||
@@ -198,24 +213,61 @@ fi
|
||||
|
||||
# --- transfer ----------------------------------------------------------------
|
||||
|
||||
# Remote paths are interpolated into shell command strings, so quote them for the
|
||||
# remote shell rather than trusting them to contain nothing surprising.
|
||||
q_remote_root=$(printf '%q' "$REMOTE_ROOT")
|
||||
q_remote_output=$(printf '%q' "$REMOTE_BUILD_OUTPUT")
|
||||
q_credentials=$(printf '%q' "$CREDENTIALS")
|
||||
|
||||
print_info "Checking host..."
|
||||
if ! ssh "$HOST" "test -d $REMOTE_ROOT"; then
|
||||
if ! ssh "$HOST" "test -d $q_remote_root"; then
|
||||
print_error "Repository not found on host: $REMOTE_ROOT"
|
||||
print_warning "Pass --remote-root if it lives elsewhere"
|
||||
exit 1
|
||||
fi
|
||||
ssh "$HOST" "mkdir -p $REMOTE_BUILD_OUTPUT"
|
||||
ssh "$HOST" "mkdir -p $q_remote_output"
|
||||
|
||||
# upload-prebuilt signs and promotes everything in the host's build-output, not
|
||||
# just what we are about to send. Anything already sitting there — typically the
|
||||
# leftovers of an earlier failed push — would ride along unnoticed.
|
||||
staged=$(ssh "$HOST" "cd $q_remote_output && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig\$' || true")
|
||||
unexpected=""
|
||||
if [[ -n "$staged" ]]; then
|
||||
while IFS= read -r remote_file; do
|
||||
[[ -z "$remote_file" ]] && continue
|
||||
for file in "${FILES[@]}"; do
|
||||
[[ "$remote_file" == "$file" ]] && continue 2
|
||||
done
|
||||
unexpected+="$remote_file"$'\n'
|
||||
done <<<"$staged"
|
||||
fi
|
||||
|
||||
if [[ -n "$unexpected" && "$INCLUDE_STAGED" != true ]]; then
|
||||
print_error "The host already has staged packages this push did not build:"
|
||||
echo ""
|
||||
echo "$unexpected" | grep -v '^$' | sed 's/^/ /'
|
||||
echo ""
|
||||
echo "Publishing signs and promotes everything in $REMOTE_BUILD_OUTPUT, so these"
|
||||
echo "would be published too. They are usually left over from a failed push."
|
||||
echo ""
|
||||
echo "Remove them on the host, or pass --include-staged to publish them as well."
|
||||
exit 1
|
||||
fi
|
||||
print_success "Host ready"
|
||||
echo ""
|
||||
|
||||
print_info "Uploading packages..."
|
||||
(cd "$BUILD_OUTPUT_DIR" && rsync -a --info=progress2 --partial "${FILES[@]}" "$HOST:$REMOTE_BUILD_OUTPUT/")
|
||||
# Prefix with ./ so rsync does not read an epoch's colon (spotify-1:1.2.3-...)
|
||||
# as a host:path separator.
|
||||
rsync_sources=()
|
||||
for file in "${FILES[@]}"; do rsync_sources+=("./$file"); done
|
||||
(cd "$BUILD_OUTPUT_DIR" && rsync -a --info=progress2 --partial "${rsync_sources[@]}" "$HOST:$REMOTE_BUILD_OUTPUT/")
|
||||
print_success "Upload complete"
|
||||
echo ""
|
||||
|
||||
print_info "Verifying checksums..."
|
||||
local_sums=$(cd "$BUILD_OUTPUT_DIR" && sha256sum "${FILES[@]}" | sort)
|
||||
remote_sums=$(ssh "$HOST" "cd $REMOTE_BUILD_OUTPUT && sha256sum $(printf '%q ' "${FILES[@]}")" | sort)
|
||||
remote_sums=$(ssh "$HOST" "cd $q_remote_output && sha256sum $(printf '%q ' "${FILES[@]}")" | sort)
|
||||
if [[ "$local_sums" != "$remote_sums" ]]; then
|
||||
print_error "Checksum mismatch after upload"
|
||||
diff <(echo "$local_sums") <(echo "$remote_sums") || true
|
||||
@@ -228,7 +280,7 @@ echo ""
|
||||
|
||||
print_info "Publishing on $HOST (sign -> promote -> update -> sync)..."
|
||||
echo ""
|
||||
if ! ssh "$HOST" "source $CREDENTIALS && cd $REMOTE_ROOT && bin/upload-prebuilt --mirror $MIRROR --arch $ARCH --skip-prod-check"; then
|
||||
if ! ssh "$HOST" "source $q_credentials && cd $q_remote_root && bin/upload-prebuilt --mirror $(printf '%q' "$MIRROR") --arch $(printf '%q' "$ARCH") --skip-prod-check"; then
|
||||
print_error "Remote publish failed"
|
||||
print_warning "The uploaded packages are still in $REMOTE_BUILD_OUTPUT on $HOST"
|
||||
exit 1
|
||||
|
||||
Reference in new issue
Block a user