David Heinemeier HanssonandClaude Opus 5
f8c1cbb072
The sync guard could not read the repository database because bsdtar was not installed on the host, and the first fix was to parse around its absence. The better answer is for the host to have what the tooling needs: libarchive ships the library pacman links against without necessarily installing the binary, so bsdtar being present was an assumption, not a fact. bin/setup installs the dependencies, enables Docker, creates the state directory, and installs and enables the release timers -- the steps the README previously listed by hand. It is idempotent and takes --check to report without changing anything. Signing credentials and the rclone remote hold secrets, so it reports on those rather than creating them. sync-repo goes back to reading the database with bsdtar alone, and says to run bin/setup when it is missing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
246 lines
8.4 KiB
Bash
Executable File
246 lines
8.4 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Source common functions
|
|
BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
|
source "$BUILD_ROOT/helpers/message-helpers.sh"
|
|
source "$BUILD_ROOT/helpers/paths.sh"
|
|
|
|
# Default remote (production)
|
|
DEFAULT_REMOTE="pkgs.omarchy.org:omarchy-pkgs"
|
|
REMOTE="$DEFAULT_REMOTE"
|
|
SKIP_PROD_CHECK=false
|
|
PRUNE=false
|
|
|
|
# Print header
|
|
print_header "Sync Repository to Remote"
|
|
|
|
# This script has no `set -e`, so a `shift 2` past the end of the argument list
|
|
# fails without consuming anything and the loop spins forever. Check first.
|
|
require_value() {
|
|
if [[ $# -lt 2 || -z "$2" ]]; then
|
|
print_error "Option $1 requires a value"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Parse arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--arch)
|
|
require_value "$@"
|
|
ARCH="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--mirror)
|
|
require_value "$@"
|
|
MIRROR="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--remote)
|
|
require_value "$@"
|
|
REMOTE="$2"
|
|
shift 2
|
|
;;
|
|
--skip-prod-check)
|
|
SKIP_PROD_CHECK=true
|
|
shift
|
|
;;
|
|
--prune)
|
|
PRUNE=true
|
|
shift
|
|
;;
|
|
-h | --help)
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --arch <arch> Target architecture (x86_64 or aarch64, default: x86_64)"
|
|
echo " --mirror <mirror> Mirror to use (edge or stable, default: edge)"
|
|
echo " --remote <remote> Rclone remote destination (default: $DEFAULT_REMOTE)"
|
|
echo " --skip-prod-check Skip production sync confirmation"
|
|
echo " --prune Also delete remote packages missing locally"
|
|
echo " -h, --help Show this help message"
|
|
echo ""
|
|
echo "Uploads are additive by default. Removing packages from the remote"
|
|
echo "requires --prune, which only makes sense from a complete local tree."
|
|
exit 0
|
|
;;
|
|
*)
|
|
print_error "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
print_info "Mirror: $MIRROR"
|
|
print_info "Architecture: $ARCH"
|
|
print_info "Local Repository: $REPO_DIR"
|
|
print_info "Remote: $REMOTE"
|
|
echo ""
|
|
|
|
# Check if local repo exists
|
|
if [[ ! -d "$REPO_DIR" ]]; then
|
|
print_error "Local repository directory not found: $REPO_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
DESTINATION_DIRECTORY="$MIRROR/$ARCH"
|
|
|
|
# Check if syncing to production and warn user
|
|
if [[ "$REMOTE" == "$DEFAULT_REMOTE" ]] && [[ "$SKIP_PROD_CHECK" != true ]]; then
|
|
print_warning "You are about to sync to PRODUCTION ($REMOTE/$DESTINATION_DIRECTORY)"
|
|
echo ""
|
|
read -p "Are you sure you want to sync to production? (y/N) " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
print_info "Sync cancelled"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
print_info "Syncing to: $REMOTE/$DESTINATION_DIRECTORY"
|
|
|
|
# The database is what users actually resolve against, and repo-add builds it
|
|
# from this tree alone. Publishing one built from a partial tree hides every
|
|
# package it does not know about, even though the files are still on the remote.
|
|
# Refuse to shrink the package list unless that is the stated intent.
|
|
#
|
|
# Compare package names, not file counts: this tree keeps several versions of
|
|
# each package (bin/repo clean --keep 2) while the database carries one entry per
|
|
# name, so counting files would compare unrelated quantities and let a partial
|
|
# tree through whenever its spare versions made up the difference.
|
|
strip_version() { sed -E 's/-[^-]+-[^-]+-[^-]+\.pkg\.tar\.[^.]+$//'; }
|
|
|
|
LOCAL_NAMES=$(ls -1 "$REPO_DIR" 2>/dev/null | grep -v '\.sig$' | grep '\.pkg\.tar\.' |
|
|
strip_version | sort -u)
|
|
|
|
# Distinguish "no repository there yet" from "cannot read the repository". Only
|
|
# the first is safe to treat as an empty remote; failing open on a credential or
|
|
# network error is how a partial database reaches production.
|
|
REMOTE_LISTING=$(rclone lsf "$REMOTE/$DESTINATION_DIRECTORY/" --s3-no-head 2>&1)
|
|
RCLONE_STATUS=$?
|
|
|
|
# rclone exit 3 is "directory not found", which is what a mirror that has never
|
|
# been published looks like. Every other failure means the remote could not be
|
|
# read, and an unread remote must not be mistaken for an empty one.
|
|
if [[ $RCLONE_STATUS -eq 3 ]]; then
|
|
REMOTE_LISTING=""
|
|
elif [[ $RCLONE_STATUS -ne 0 ]]; then
|
|
print_error "Cannot read the remote repository (rclone exit $RCLONE_STATUS)"
|
|
echo "$REMOTE_LISTING"
|
|
echo ""
|
|
echo "Refusing to sync: an unreadable remote cannot be checked for packages"
|
|
echo "this tree would hide."
|
|
exit 1
|
|
fi
|
|
|
|
if grep -qx 'omarchy\.db' <<<"$REMOTE_LISTING"; then
|
|
# bsdtar, not tar: the database is compressed and GNU tar will not detect that
|
|
# on a pipe. repo-add has used both gzip and zstd, so let libarchive decide.
|
|
REMOTE_DB_FILE=$(mktemp)
|
|
trap 'rm -f "$REMOTE_DB_FILE"' EXIT
|
|
rclone cat "$REMOTE/$DESTINATION_DIRECTORY/omarchy.db" --s3-no-head >"$REMOTE_DB_FILE" 2>/dev/null
|
|
|
|
if ! command -v bsdtar >/dev/null 2>&1; then
|
|
print_error "bsdtar is not installed"
|
|
echo ""
|
|
echo "The repository host needs it to read the repository database."
|
|
echo "Run bin/setup to install everything this host requires."
|
|
exit 1
|
|
fi
|
|
|
|
# bsdtar, not tar: repo-add has used both gzip and zstd for the database, and
|
|
# libarchive detects either without being told which.
|
|
REMOTE_NAMES=$(bsdtar -tf "$REMOTE_DB_FILE" 2>/dev/null | sed 's|/.*||' |
|
|
sed -E 's/-[^-]+-[^-]+$//' | sort -u)
|
|
|
|
if [[ -z "$REMOTE_NAMES" ]]; then
|
|
print_error "The remote database exists but could not be read"
|
|
echo ""
|
|
echo "Refusing to sync rather than assume the remote is empty. Check that"
|
|
echo "omarchy.db is not corrupt:"
|
|
echo " rclone cat $REMOTE/$DESTINATION_DIRECTORY/omarchy.db | bsdtar -tf -"
|
|
exit 1
|
|
fi
|
|
|
|
HIDDEN=$(comm -23 <(echo "$REMOTE_NAMES") <(echo "$LOCAL_NAMES"))
|
|
HIDDEN_COUNT=$(grep -c '' <<<"$HIDDEN")
|
|
[[ -z "$HIDDEN" ]] && HIDDEN_COUNT=0
|
|
|
|
if [[ "$HIDDEN_COUNT" -gt 0 && "$PRUNE" != true ]]; then
|
|
print_error "$HIDDEN_COUNT package(s) in the remote database are missing from this tree"
|
|
echo ""
|
|
echo "$HIDDEN" | head -10 | sed 's/^/ /'
|
|
[[ "$HIDDEN_COUNT" -gt 10 ]] && echo " ... and $((HIDDEN_COUNT - 10)) more"
|
|
echo ""
|
|
echo "Publishing a database built here would hide them, even though their"
|
|
echo "files remain on the mirror."
|
|
echo ""
|
|
echo "To publish packages built on this machine, push them to the build host,"
|
|
echo "which holds the complete repository:"
|
|
echo " bin/repo push --mirror $MIRROR --arch $ARCH"
|
|
echo ""
|
|
echo "If shrinking the repository is genuinely what you want, pass --prune."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Upload packages first, database last, so the remote never advertises a package
|
|
# it does not yet have.
|
|
#
|
|
# This is `copy`, not `sync`: a local tree is not authoritative about what should
|
|
# exist on the remote. Packages built on another machine live only in that
|
|
# machine's build-output, and pkgs.omarchy.org/ is gitignored, so any checkout
|
|
# that has not run a full release is missing nearly everything. `sync` would read
|
|
# those absences as deletions and empty the repository. --ignore-existing keeps
|
|
# versions already published from being overwritten.
|
|
if [[ "$PRUNE" == true ]]; then
|
|
print_warning "Pruning: remote packages missing from $REPO_DIR will be DELETED"
|
|
if [[ "$SKIP_PROD_CHECK" != true ]]; then
|
|
read -p "Prune the remote to match this tree? (y/N) " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
print_info "Sync cancelled"
|
|
exit 0
|
|
fi
|
|
fi
|
|
print_info "Syncing packages (with prune)..."
|
|
if ! rclone sync "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
|
--s3-no-head \
|
|
--exclude "omarchy.db*" \
|
|
--exclude "omarchy.files*" \
|
|
--ignore-existing \
|
|
--copy-links --delete-after -v; then
|
|
print_error "Package sync failed — not publishing the database"
|
|
exit 1
|
|
fi
|
|
else
|
|
print_info "Syncing packages..."
|
|
if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
|
--s3-no-head \
|
|
--exclude "omarchy.db*" \
|
|
--exclude "omarchy.files*" \
|
|
--ignore-existing \
|
|
--copy-links -v; then
|
|
print_error "Package upload failed — not publishing the database"
|
|
print_warning "The remote database still describes the previous contents, so"
|
|
print_warning "the repository is unchanged and consistent."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Then sync database files last to ensure repository integrity
|
|
print_info "Updating repository database..."
|
|
if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \
|
|
--s3-no-head \
|
|
--include "omarchy.*" \
|
|
--checksum --copy-links -v; then
|
|
print_error "Database upload failed"
|
|
print_warning "Packages were uploaded but the database still describes the"
|
|
print_warning "previous contents. Re-run sync to finish publishing them."
|
|
exit 1
|
|
fi
|
|
|
|
print_success "Sync complete!"
|