David Heinemeier HanssonandClaude Opus 5
dbe1db4b03
The partial-tree guard parsed omarchy.db with bsdtar, which is not installed on the repository host. Every sync there aborted with "the remote database exists but could not be read" -- a guard meant to catch a partial tree instead blocked a complete one, stopping a publish after sign, promote and update had already succeeded. GNU tar reads the database fine when it is a seekable file; the pipe was what defeated it originally, and that is already downloaded to a temp file. tar now leads, with bsdtar as a fallback for a tar too old to detect zstd. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
244 lines
8.5 KiB
Bash
Executable File
244 lines
8.5 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Source common functions
|
|
BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
|
source "$BUILD_ROOT/helpers/message-helpers.sh"
|
|
source "$BUILD_ROOT/helpers/paths.sh"
|
|
|
|
# Default remote (production)
|
|
DEFAULT_REMOTE="pkgs.omarchy.org:omarchy-pkgs"
|
|
REMOTE="$DEFAULT_REMOTE"
|
|
SKIP_PROD_CHECK=false
|
|
PRUNE=false
|
|
|
|
# Print header
|
|
print_header "Sync Repository to Remote"
|
|
|
|
# This script has no `set -e`, so a `shift 2` past the end of the argument list
|
|
# fails without consuming anything and the loop spins forever. Check first.
|
|
require_value() {
|
|
if [[ $# -lt 2 || -z "$2" ]]; then
|
|
print_error "Option $1 requires a value"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--arch)
|
|
require_value "$@"
|
|
ARCH="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--mirror)
|
|
require_value "$@"
|
|
MIRROR="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--remote)
|
|
require_value "$@"
|
|
REMOTE="$2"
|
|
shift 2
|
|
;;
|
|
--skip-prod-check)
|
|
SKIP_PROD_CHECK=true
|
|
shift
|
|
;;
|
|
--prune)
|
|
PRUNE=true
|
|
shift
|
|
;;
|
|
-h | --help)
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --arch <arch> Target architecture (x86_64 or aarch64, default: x86_64)"
|
|
echo " --mirror <mirror> Mirror to use (edge or stable, default: edge)"
|
|
echo " --remote <remote> Rclone remote destination (default: $DEFAULT_REMOTE)"
|
|
echo " --skip-prod-check Skip production sync confirmation"
|
|
echo " --prune Also delete remote packages missing locally"
|
|
echo " -h, --help Show this help message"
|
|
echo ""
|
|
echo "Uploads are additive by default. Removing packages from the remote"
|
|
echo "requires --prune, which only makes sense from a complete local tree."
|
|
exit 0
|
|
;;
|
|
*)
|
|
print_error "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
print_info "Mirror: $MIRROR"
|
|
print_info "Architecture: $ARCH"
|
|
print_info "Local Repository: $REPO_DIR"
|
|
print_info "Remote: $REMOTE"
|
|
echo ""
|
|
|
|
# Check if local repo exists
|
|
if [[ ! -d "$REPO_DIR" ]]; then
|
|
print_error "Local repository directory not found: $REPO_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
DESTINATION_DIRECTORY="$MIRROR/$ARCH"
|
|
|
|
# Check if syncing to production and warn user
|
|
if [[ "$REMOTE" == "$DEFAULT_REMOTE" ]] && [[ "$SKIP_PROD_CHECK" != true ]]; then
|
|
print_warning "You are about to sync to PRODUCTION ($REMOTE/$DESTINATION_DIRECTORY)"
|
|
echo ""
|
|
read -p "Are you sure you want to sync to production? (y/N) " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
print_info "Sync cancelled"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
print_info "Syncing to: $REMOTE/$DESTINATION_DIRECTORY"
|
|
|
|
# The database is what users actually resolve against, and repo-add builds it
|
|
# from this tree alone. Publishing one built from a partial tree hides every
|
|
# package it does not know about, even though the files are still on the remote.
|
|
# Refuse to shrink the package list unless that is the stated intent.
|
|
#
|
|
# Compare package names, not file counts: this tree keeps several versions of
|
|
# each package (bin/repo clean --keep 2) while the database carries one entry per
|
|
# name, so counting files would compare unrelated quantities and let a partial
|
|
# tree through whenever its spare versions made up the difference.
|
|
strip_version() { sed -E 's/-[^-]+-[^-]+-[^-]+\.pkg\.tar\.[^.]+$//'; }
|
|
|
|
LOCAL_NAMES=$(ls -1 "$REPO_DIR" 2>/dev/null | grep -v '\.sig$' | grep '\.pkg\.tar\.' |
|
|
strip_version | sort -u)
|
|
|
|
# Distinguish "no repository there yet" from "cannot read the repository". Only
|
|
# the first is safe to treat as an empty remote; failing open on a credential or
|
|
# network error is how a partial database reaches production.
|
|
REMOTE_LISTING=$(rclone lsf "$REMOTE/$DESTINATION_DIRECTORY/" --s3-no-head 2>&1)
|
|
RCLONE_STATUS=$?
|
|
|
|
# rclone exit 3 is "directory not found", which is what a mirror that has never
|
|
# been published looks like. Every other failure means the remote could not be
|
|
# read, and an unread remote must not be mistaken for an empty one.
|
|
if [[ $RCLONE_STATUS -eq 3 ]]; then
|
|
REMOTE_LISTING=""
|
|
elif [[ $RCLONE_STATUS -ne 0 ]]; then
|
|
print_error "Cannot read the remote repository (rclone exit $RCLONE_STATUS)"
|
|
echo "$REMOTE_LISTING"
|
|
echo ""
|
|
echo "Refusing to sync: an unreadable remote cannot be checked for packages"
|
|
echo "this tree would hide."
|
|
exit 1
|
|
fi
|
|
|
|
if grep -qx 'omarchy\.db' <<<"$REMOTE_LISTING"; then
|
|
# bsdtar, not tar: the database is compressed and GNU tar will not detect that
|
|
# on a pipe. repo-add has used both gzip and zstd, so let libarchive decide.
|
|
REMOTE_DB_FILE=$(mktemp)
|
|
trap 'rm -f "$REMOTE_DB_FILE"' EXIT
|
|
rclone cat "$REMOTE/$DESTINATION_DIRECTORY/omarchy.db" --s3-no-head >"$REMOTE_DB_FILE" 2>/dev/null
|
|
|
|
# Download to a file rather than piping: GNU tar detects the compression from
|
|
# a seekable archive but not from a pipe, and the database has been both gzip
|
|
# and zstd over its life. bsdtar reads either too, but it is not present on
|
|
# every host, so tar leads and bsdtar covers a tar too old for zstd.
|
|
REMOTE_ENTRIES=$(tar -tf "$REMOTE_DB_FILE" 2>/dev/null)
|
|
if [[ -z "$REMOTE_ENTRIES" ]] && command -v bsdtar >/dev/null 2>&1; then
|
|
REMOTE_ENTRIES=$(bsdtar -tf "$REMOTE_DB_FILE" 2>/dev/null)
|
|
fi
|
|
|
|
REMOTE_NAMES=$(printf '%s\n' "$REMOTE_ENTRIES" | sed 's|/.*||' |
|
|
sed -E 's/-[^-]+-[^-]+$//' | sort -u | grep -v '^$')
|
|
|
|
if [[ -z "$REMOTE_NAMES" ]]; then
|
|
print_error "The remote database exists but could not be read"
|
|
echo ""
|
|
echo "Refusing to sync rather than assume the remote is empty. Check that"
|
|
echo "omarchy.db is not corrupt: rclone cat $REMOTE/$DESTINATION_DIRECTORY/omarchy.db | tar -t"
|
|
exit 1
|
|
fi
|
|
|
|
HIDDEN=$(comm -23 <(echo "$REMOTE_NAMES") <(echo "$LOCAL_NAMES"))
|
|
HIDDEN_COUNT=$(grep -c '' <<<"$HIDDEN")
|
|
[[ -z "$HIDDEN" ]] && HIDDEN_COUNT=0
|
|
|
|
if [[ "$HIDDEN_COUNT" -gt 0 && "$PRUNE" != true ]]; then
|
|
print_error "$HIDDEN_COUNT package(s) in the remote database are missing from this tree"
|
|
echo ""
|
|
echo "$HIDDEN" | head -10 | sed 's/^/ /'
|
|
[[ "$HIDDEN_COUNT" -gt 10 ]] && echo " ... and $((HIDDEN_COUNT - 10)) more"
|
|
echo ""
|
|
echo "Publishing a database built here would hide them, even though their"
|
|
echo "files remain on the mirror."
|
|
echo ""
|
|
echo "To publish packages built on this machine, push them to the build host,"
|
|
echo "which holds the complete repository:"
|
|
echo " bin/repo push --mirror $MIRROR --arch $ARCH"
|
|
echo ""
|
|
echo "If shrinking the repository is genuinely what you want, pass --prune."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Upload packages first, database last, so the remote never advertises a package
|
|
# it does not yet have.
|
|
#
|
|
# This is `copy`, not `sync`: a local tree is not authoritative about what should
|
|
# exist on the remote. Packages built on another machine live only in that
|
|
# machine's build-output, and pkgs.omarchy.org/ is gitignored, so any checkout
|
|
# that has not run a full release is missing nearly everything. `sync` would read
|
|
# those absences as deletions and empty the repository. --ignore-existing keeps
|
|
# versions already published from being overwritten.
|
|
if [[ "$PRUNE" == true ]]; then
|
|
print_warning "Pruning: remote packages missing from $REPO_DIR will be DELETED"
|
|
if [[ "$SKIP_PROD_CHECK" != true ]]; then
|
|
read -p "Prune the remote to match this tree? (y/N) " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
print_info "Sync cancelled"
|
|
exit 0
|
|
fi
|
|
fi
|
|
print_info "Syncing packages (with prune)..."
|
|
if ! rclone sync "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
|
--s3-no-head \
|
|
--exclude "omarchy.db*" \
|
|
--exclude "omarchy.files*" \
|
|
--ignore-existing \
|
|
--copy-links --delete-after -v; then
|
|
print_error "Package sync failed — not publishing the database"
|
|
exit 1
|
|
fi
|
|
else
|
|
print_info "Syncing packages..."
|
|
if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
|
--s3-no-head \
|
|
--exclude "omarchy.db*" \
|
|
--exclude "omarchy.files*" \
|
|
--ignore-existing \
|
|
--copy-links -v; then
|
|
print_error "Package upload failed — not publishing the database"
|
|
print_warning "The remote database still describes the previous contents, so"
|
|
print_warning "the repository is unchanged and consistent."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Then sync database files last to ensure repository integrity
|
|
print_info "Updating repository database..."
|
|
if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
|
--s3-no-head \
|
|
--include "omarchy.*" \
|
|
--checksum --copy-links -v; then
|
|
print_error "Database upload failed"
|
|
print_warning "Packages were uploaded but the database still describes the"
|
|
print_warning "previous contents. Re-run sync to finish publishing them."
|
|
exit 1
|
|
fi
|
|
|
|
print_success "Sync complete!"
|