Fix nine defects in the push/sync path found in review
The worst was fatal: push passed --skip-prod-check to upload-prebuilt, which
forwards every argument to sign, promote and update as well, and sign rejects
unknown options. Every non-dry-run push and deploy would have uploaded and
verified its artifacts and then failed before signing. upload-prebuilt now
routes publishing flags to sync alone.
The partial-tree guard was weaker than it looked:
- it counted archive files locally against package names in the remote
database, and this tree keeps two versions per package, so a checkout with
a spare version of half the repository could pass while still hiding
hundreds of packages. It now compares package-name sets and lists what
would be hidden.
- it treated any unreadable remote as an empty one, so an auth failure or a
corrupt database disabled it. Only rclone's "directory not found" now
counts as a fresh mirror; every other failure aborts.
Also:
- sync had no set -e, so a failed package upload fell through to publishing
the database, advertising packages that were never uploaded. Each transfer
is now checked before the next step.
- --package with no names silently meant "every package", which under --yes
could publish everything from one unset variable in a script.
- push now refuses to run when the host has packages staged from an earlier
failure, since publishing would sign and promote those too.
- epoch versions contain a colon, which rsync reads as host:path, so no
package with an epoch could be transferred. Sources are ./-prefixed.
- remote paths are quoted for the remote shell.
- sync spun forever on a missing option value.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c5f5f1c12c
commit
fd9c078bf2
5 files changed
+192
-33
No files matched your search
+95
-21
@@ -14,20 +14,32 @@ PRUNE=false
|
||||
# Print header
|
||||
print_header "Sync Repository to Remote"
|
||||
|
||||
# This script has no `set -e`, so a `shift 2` past the end of the argument list
|
||||
# fails without consuming anything and the loop spins forever. Check first.
|
||||
require_value() {
|
||||
if [[ $# -lt 2 || -z "$2" ]]; then
|
||||
print_error "Option $1 requires a value"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--arch)
|
||||
require_value "$@"
|
||||
ARCH="$2"
|
||||
update_arch_paths
|
||||
shift 2
|
||||
;;
|
||||
--mirror)
|
||||
require_value "$@"
|
||||
MIRROR="$2"
|
||||
update_arch_paths
|
||||
shift 2
|
||||
;;
|
||||
--remote)
|
||||
require_value "$@"
|
||||
REMOTE="$2"
|
||||
shift 2
|
||||
;;
|
||||
@@ -93,26 +105,75 @@ print_info "Syncing to: $REMOTE/$DESTINATION_DIRECTORY"
|
||||
# from this tree alone. Publishing one built from a partial tree hides every
|
||||
# package it does not know about, even though the files are still on the remote.
|
||||
# Refuse to shrink the package list unless that is the stated intent.
|
||||
LOCAL_COUNT=$(ls -1 "$REPO_DIR"/*.pkg.tar.* 2>/dev/null | grep -vc '\.sig$' || true)
|
||||
# bsdtar, not tar: the database is compressed and GNU tar will not detect that
|
||||
# on a pipe. repo-add has used both gzip and zstd, so let libarchive decide.
|
||||
REMOTE_COUNT=$(rclone cat "$REMOTE/$DESTINATION_DIRECTORY/omarchy.db" --s3-no-head 2>/dev/null |
|
||||
bsdtar -tf - 2>/dev/null | sed 's|/.*||' | sort -u | grep -c . || true)
|
||||
#
|
||||
# Compare package names, not file counts: this tree keeps several versions of
|
||||
# each package (bin/repo clean --keep 2) while the database carries one entry per
|
||||
# name, so counting files would compare unrelated quantities and let a partial
|
||||
# tree through whenever its spare versions made up the difference.
|
||||
strip_version() { sed -E 's/-[^-]+-[^-]+-[^-]+\.pkg\.tar\.[^.]+$//'; }
|
||||
|
||||
if [[ "${REMOTE_COUNT:-0}" -gt 0 && "${LOCAL_COUNT:-0}" -lt "$REMOTE_COUNT" && "$PRUNE" != true ]]; then
|
||||
print_error "Local tree has $LOCAL_COUNT package(s); the remote database lists $REMOTE_COUNT"
|
||||
LOCAL_NAMES=$(ls -1 "$REPO_DIR" 2>/dev/null | grep -v '\.sig$' | grep '\.pkg\.tar\.' |
|
||||
strip_version | sort -u)
|
||||
|
||||
# Distinguish "no repository there yet" from "cannot read the repository". Only
|
||||
# the first is safe to treat as an empty remote; failing open on a credential or
|
||||
# network error is how a partial database reaches production.
|
||||
REMOTE_LISTING=$(rclone lsf "$REMOTE/$DESTINATION_DIRECTORY/" --s3-no-head 2>&1)
|
||||
RCLONE_STATUS=$?
|
||||
|
||||
# rclone exit 3 is "directory not found", which is what a mirror that has never
|
||||
# been published looks like. Every other failure means the remote could not be
|
||||
# read, and an unread remote must not be mistaken for an empty one.
|
||||
if [[ $RCLONE_STATUS -eq 3 ]]; then
|
||||
REMOTE_LISTING=""
|
||||
elif [[ $RCLONE_STATUS -ne 0 ]]; then
|
||||
print_error "Cannot read the remote repository (rclone exit $RCLONE_STATUS)"
|
||||
echo "$REMOTE_LISTING"
|
||||
echo ""
|
||||
echo "Publishing this database would hide the $((REMOTE_COUNT - LOCAL_COUNT)) package(s)"
|
||||
echo "missing from $REPO_DIR."
|
||||
echo ""
|
||||
echo "To publish packages built on this machine, push them to the build host,"
|
||||
echo "which holds the complete repository:"
|
||||
echo " bin/repo push --mirror $MIRROR --arch $ARCH"
|
||||
echo ""
|
||||
echo "If shrinking the repository is genuinely what you want, pass --prune."
|
||||
echo "Refusing to sync: an unreadable remote cannot be checked for packages"
|
||||
echo "this tree would hide."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if grep -qx 'omarchy\.db' <<<"$REMOTE_LISTING"; then
|
||||
# bsdtar, not tar: the database is compressed and GNU tar will not detect that
|
||||
# on a pipe. repo-add has used both gzip and zstd, so let libarchive decide.
|
||||
REMOTE_DB_FILE=$(mktemp)
|
||||
trap 'rm -f "$REMOTE_DB_FILE"' EXIT
|
||||
rclone cat "$REMOTE/$DESTINATION_DIRECTORY/omarchy.db" --s3-no-head >"$REMOTE_DB_FILE" 2>/dev/null
|
||||
REMOTE_NAMES=$(bsdtar -tf "$REMOTE_DB_FILE" 2>/dev/null | sed 's|/.*||' |
|
||||
sed -E 's/-[^-]+-[^-]+$//' | sort -u)
|
||||
|
||||
if [[ -z "$REMOTE_NAMES" ]]; then
|
||||
print_error "The remote database exists but could not be read"
|
||||
echo ""
|
||||
echo "Refusing to sync rather than assume the remote is empty. Check that"
|
||||
echo "bsdtar is installed and that omarchy.db is not corrupt."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
HIDDEN=$(comm -23 <(echo "$REMOTE_NAMES") <(echo "$LOCAL_NAMES"))
|
||||
HIDDEN_COUNT=$(grep -c '' <<<"$HIDDEN")
|
||||
[[ -z "$HIDDEN" ]] && HIDDEN_COUNT=0
|
||||
|
||||
if [[ "$HIDDEN_COUNT" -gt 0 && "$PRUNE" != true ]]; then
|
||||
print_error "$HIDDEN_COUNT package(s) in the remote database are missing from this tree"
|
||||
echo ""
|
||||
echo "$HIDDEN" | head -10 | sed 's/^/ /'
|
||||
[[ "$HIDDEN_COUNT" -gt 10 ]] && echo " ... and $((HIDDEN_COUNT - 10)) more"
|
||||
echo ""
|
||||
echo "Publishing a database built here would hide them, even though their"
|
||||
echo "files remain on the mirror."
|
||||
echo ""
|
||||
echo "To publish packages built on this machine, push them to the build host,"
|
||||
echo "which holds the complete repository:"
|
||||
echo " bin/repo push --mirror $MIRROR --arch $ARCH"
|
||||
echo ""
|
||||
echo "If shrinking the repository is genuinely what you want, pass --prune."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Upload packages first, database last, so the remote never advertises a package
|
||||
# it does not yet have.
|
||||
#
|
||||
@@ -133,27 +194,40 @@ if [[ "$PRUNE" == true ]]; then
|
||||
fi
|
||||
fi
|
||||
print_info "Syncing packages (with prune)..."
|
||||
rclone sync "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
||||
if ! rclone sync "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
||||
--s3-no-head \
|
||||
--exclude "omarchy.db*" \
|
||||
--exclude "omarchy.files*" \
|
||||
--ignore-existing \
|
||||
--copy-links --delete-after -v
|
||||
--copy-links --delete-after -v; then
|
||||
print_error "Package sync failed — not publishing the database"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
print_info "Syncing packages..."
|
||||
rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
||||
if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
||||
--s3-no-head \
|
||||
--exclude "omarchy.db*" \
|
||||
--exclude "omarchy.files*" \
|
||||
--ignore-existing \
|
||||
--copy-links -v
|
||||
--copy-links -v; then
|
||||
print_error "Package upload failed — not publishing the database"
|
||||
print_warning "The remote database still describes the previous contents, so"
|
||||
print_warning "the repository is unchanged and consistent."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Then sync database files last to ensure repository integrity
|
||||
print_info "Updating repository database..."
|
||||
rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
||||
if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
||||
--s3-no-head \
|
||||
--include "omarchy.*" \
|
||||
--checksum --copy-links -v
|
||||
--checksum --copy-links -v; then
|
||||
print_error "Database upload failed"
|
||||
print_warning "Packages were uploaded but the database still describes the"
|
||||
print_warning "previous contents. Re-run sync to finish publishing them."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
print_success "Sync complete!"
|
||||
Reference in new issue
Block a user