David Heinemeier HanssonandClaude Opus 5
ca90a19d73
--package meant a pkgbase to bin/build and a literal package name to bin/push-build, so deploy --package nvidia-580xx-utils built three packages and published one, leaving nvidia-580xx-dkms and opencl-nvidia-580xx behind with no indication anything was missing. Hit while deploying exactly that package. Selection now matches on the pkgbase recorded in .PKGINFO as well as on the package name, so a pkgbase ships all of its outputs and an individual name still selects just that one. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
314 lines
10 KiB
Bash
Executable File
314 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
# Push locally built packages to the repository host and publish them there.
|
|
#
|
|
# Heavy packages are quicker to build on a local machine than on the server, but
|
|
# publishing has to happen where the full repository lives: the signing key is on
|
|
# the repository host, and `bin/repo sync` can only produce a correct remote from a
|
|
# complete local tree. So this uploads the artifacts and runs the publish steps
|
|
# over ssh rather than syncing from here.
|
|
|
|
set -e
|
|
|
|
BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
|
source "$BUILD_ROOT/helpers/message-helpers.sh"
|
|
source "$BUILD_ROOT/helpers/paths.sh"
|
|
source "$BUILD_ROOT/helpers/host-helpers.sh"
|
|
|
|
HOST=""
|
|
REMOTE_ROOT="/root/omarchy-pkgs"
|
|
CREDENTIALS="/root/.omarchy/build-credentials"
|
|
PACKAGES=""
|
|
PACKAGE_FLAG_GIVEN=false
|
|
DRY_RUN=false
|
|
ASSUME_YES=false
|
|
INCLUDE_STAGED=false
|
|
|
|
print_header "Push Build to Host"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--arch)
|
|
ARCH="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--mirror)
|
|
MIRROR="$2"
|
|
if [[ "$MIRROR" != "edge" && "$MIRROR" != "stable" ]]; then
|
|
print_error "Invalid mirror: $MIRROR (must be 'edge' or 'stable')"
|
|
exit 1
|
|
fi
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--package)
|
|
shift
|
|
PACKAGE_FLAG_GIVEN=true
|
|
while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do
|
|
[[ -n "$1" ]] && PACKAGES="$PACKAGES $1"
|
|
shift
|
|
done
|
|
PACKAGES="${PACKAGES# }"
|
|
;;
|
|
--host)
|
|
HOST="$2"
|
|
shift 2
|
|
;;
|
|
--remote-root)
|
|
REMOTE_ROOT="$2"
|
|
shift 2
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-y | --yes)
|
|
ASSUME_YES=true
|
|
shift
|
|
;;
|
|
--include-staged)
|
|
INCLUDE_STAGED=true
|
|
shift
|
|
;;
|
|
-h | --help)
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Upload packages from build-output/ to the repository host, then sign,"
|
|
echo "promote, update and sync them there."
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --arch <arch> Target architecture (default: x86_64)"
|
|
echo " --mirror <mirror> Mirror to publish to (edge or stable, default: edge)"
|
|
echo " --package <names> Only push these packages (space-separated)"
|
|
echo " --host <host> ssh destination (default: \$OMARCHY_REPO_HOST or .repo-host)"
|
|
echo " --remote-root <path> Repository path on the host (default: $REMOTE_ROOT)"
|
|
echo " --dry-run Show what would be pushed, transfer nothing"
|
|
echo " -y, --yes Do not ask for confirmation"
|
|
echo " --include-staged Publish packages already staged on the host too"
|
|
echo " -h, --help Show this help message"
|
|
echo ""
|
|
echo "Typical use:"
|
|
echo " bin/repo build --package nvidia-580xx-utils"
|
|
echo " bin/repo push --package nvidia-580xx-utils"
|
|
exit 0
|
|
;;
|
|
*)
|
|
print_error "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# --- host resolution ---------------------------------------------------------
|
|
|
|
if ! HOST=$(resolve_repo_host "$HOST"); then
|
|
print_no_repo_host
|
|
exit 1
|
|
fi
|
|
|
|
# --- collect artifacts -------------------------------------------------------
|
|
|
|
if [[ ! -d "$BUILD_OUTPUT_DIR" ]]; then
|
|
print_error "Build output directory not found: $BUILD_OUTPUT_DIR"
|
|
print_warning "Run bin/repo build first"
|
|
exit 1
|
|
fi
|
|
|
|
# Package files only. Signatures are produced on the host, and the repo database
|
|
# is rebuilt there, so neither should ride along.
|
|
mapfile -t ALL_FILES < <(cd "$BUILD_OUTPUT_DIR" && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig$' || true)
|
|
|
|
# "--package" with nothing after it, or with an empty variable, must not quietly
|
|
# widen to every artifact — that is the difference between shipping one package
|
|
# and shipping whatever else happens to be lying around.
|
|
if [[ "$PACKAGE_FLAG_GIVEN" == true && -z "$PACKAGES" ]]; then
|
|
print_error "--package requires at least one package name"
|
|
exit 1
|
|
fi
|
|
|
|
# On a build machine an unscoped build leaves the whole repository in
|
|
# build-output, because there is no local database to tell it what already
|
|
# exists. Interactively that is survivable — the confirmation below lists every
|
|
# package first — but with --yes nobody sees the list, so require an explicit
|
|
# selection instead.
|
|
if [[ -z "$PACKAGES" && "$ASSUME_YES" == true ]] && ! on_repo_host; then
|
|
print_error "--package is required to publish unattended from a build machine"
|
|
echo ""
|
|
echo "There is no repository database in $REPO_DIR, so a preceding unscoped"
|
|
echo "build would have rebuilt everything rather than only what changed, and"
|
|
echo "--yes would publish all ${#ALL_FILES[@]} of them without showing the list."
|
|
echo ""
|
|
echo "Name the packages to publish:"
|
|
echo " bin/repo push --package <name>"
|
|
exit 1
|
|
fi
|
|
|
|
# --package means the same thing here as it does to bin/build: a pkgbase, whose
|
|
# every output ships together. Selecting only the artifact whose filename matched
|
|
# would publish one third of a split package like nvidia-580xx-utils and silently
|
|
# leave nvidia-580xx-dkms and opencl-nvidia-580xx behind. An output's own name
|
|
# still matches, for pushing just one of them on purpose.
|
|
#
|
|
# pkgbase comes from .PKGINFO rather than the PKGBUILD: it is what makepkg
|
|
# actually recorded, and it needs no guessing about which directory built what.
|
|
pkgbase_of() {
|
|
bsdtar -xOf "$1" .PKGINFO 2>/dev/null |
|
|
awk -F ' = ' '$1 == "pkgbase" { print $2; exit }'
|
|
}
|
|
|
|
FILES=()
|
|
if [[ -z "$PACKAGES" ]]; then
|
|
FILES=("${ALL_FILES[@]}")
|
|
else
|
|
declare -A MATCHED=()
|
|
for file in "${ALL_FILES[@]}"; do
|
|
# name-version-release-arch.pkg.tar.zst -> name
|
|
pkgname="${file%-*-*-*.pkg.tar.*}"
|
|
pkgbase=$(pkgbase_of "$BUILD_OUTPUT_DIR/$file")
|
|
for wanted in $PACKAGES; do
|
|
if [[ "$pkgname" == "$wanted" || "$pkgbase" == "$wanted" ]]; then
|
|
FILES+=("$file")
|
|
MATCHED["$wanted"]=1
|
|
break
|
|
fi
|
|
done
|
|
done
|
|
|
|
for wanted in $PACKAGES; do
|
|
if [[ -z "${MATCHED[$wanted]:-}" ]]; then
|
|
print_error "No built artifact for '$wanted' in $BUILD_OUTPUT_DIR"
|
|
print_warning "Name a package or the pkgbase it was built from"
|
|
exit 1
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [[ ${#FILES[@]} -eq 0 ]]; then
|
|
print_error "No packages found in $BUILD_OUTPUT_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
REMOTE_BUILD_OUTPUT="$REMOTE_ROOT/build-output/$MIRROR/$ARCH"
|
|
|
|
print_info "Host: $HOST"
|
|
print_info "Mirror: $MIRROR"
|
|
print_info "Architecture: $ARCH"
|
|
print_info "Local build output: $BUILD_OUTPUT_DIR"
|
|
print_info "Remote build output: $REMOTE_BUILD_OUTPUT"
|
|
echo ""
|
|
|
|
total=0
|
|
print_info "${#FILES[@]} package(s) to push:"
|
|
for file in "${FILES[@]}"; do
|
|
size=$(stat -c %s "$BUILD_OUTPUT_DIR/$file")
|
|
total=$((total + size))
|
|
print_step "$file ($(numfmt --to=iec --format %.1f "$size"))"
|
|
done
|
|
echo ""
|
|
print_info "Total transfer: $(numfmt --to=iec --format %.1f "$total")"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
print_warning "DRY RUN - nothing transferred"
|
|
echo ""
|
|
print_info "Would run on $HOST:"
|
|
echo " source $CREDENTIALS && cd $REMOTE_ROOT && bin/upload-prebuilt --mirror $MIRROR --arch $ARCH"
|
|
exit 0
|
|
fi
|
|
|
|
# Publishing reaches production, so confirm here. The remote publish runs
|
|
# non-interactively and cannot ask.
|
|
if [[ "$ASSUME_YES" != true ]]; then
|
|
print_warning "This publishes to PRODUCTION via $HOST ($MIRROR/$ARCH)"
|
|
read -p "Continue? (y/N) " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
print_info "Push cancelled"
|
|
exit 0
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
# --- transfer ----------------------------------------------------------------
|
|
|
|
# Remote paths are interpolated into shell command strings, so quote them for the
|
|
# remote shell rather than trusting them to contain nothing surprising.
|
|
q_remote_root=$(printf '%q' "$REMOTE_ROOT")
|
|
q_remote_output=$(printf '%q' "$REMOTE_BUILD_OUTPUT")
|
|
q_credentials=$(printf '%q' "$CREDENTIALS")
|
|
|
|
print_info "Checking host..."
|
|
if ! ssh "$HOST" "test -d $q_remote_root"; then
|
|
print_error "Repository not found on host: $REMOTE_ROOT"
|
|
print_warning "Pass --remote-root if it lives elsewhere"
|
|
exit 1
|
|
fi
|
|
ssh "$HOST" "mkdir -p $q_remote_output"
|
|
|
|
# upload-prebuilt signs and promotes everything in the host's build-output, not
|
|
# just what we are about to send. Anything already sitting there — typically the
|
|
# leftovers of an earlier failed push — would ride along unnoticed.
|
|
staged=$(ssh "$HOST" "cd $q_remote_output && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig\$' || true")
|
|
unexpected=""
|
|
if [[ -n "$staged" ]]; then
|
|
while IFS= read -r remote_file; do
|
|
[[ -z "$remote_file" ]] && continue
|
|
for file in "${FILES[@]}"; do
|
|
[[ "$remote_file" == "$file" ]] && continue 2
|
|
done
|
|
unexpected+="$remote_file"$'\n'
|
|
done <<<"$staged"
|
|
fi
|
|
|
|
if [[ -n "$unexpected" && "$INCLUDE_STAGED" != true ]]; then
|
|
print_error "The host already has staged packages this push did not build:"
|
|
echo ""
|
|
echo "$unexpected" | grep -v '^$' | sed 's/^/ /'
|
|
echo ""
|
|
echo "Publishing signs and promotes everything in $REMOTE_BUILD_OUTPUT, so these"
|
|
echo "would be published too. They are usually left over from a failed push."
|
|
echo ""
|
|
echo "Remove them on the host, or pass --include-staged to publish them as well."
|
|
exit 1
|
|
fi
|
|
print_success "Host ready"
|
|
echo ""
|
|
|
|
print_info "Uploading packages..."
|
|
# Prefix with ./ so rsync does not read an epoch's colon (spotify-1:1.2.3-...)
|
|
# as a host:path separator.
|
|
rsync_sources=()
|
|
for file in "${FILES[@]}"; do rsync_sources+=("./$file"); done
|
|
(cd "$BUILD_OUTPUT_DIR" && rsync -a --info=progress2 --partial "${rsync_sources[@]}" "$HOST:$REMOTE_BUILD_OUTPUT/")
|
|
print_success "Upload complete"
|
|
echo ""
|
|
|
|
print_info "Verifying checksums..."
|
|
local_sums=$(cd "$BUILD_OUTPUT_DIR" && sha256sum "${FILES[@]}" | sort)
|
|
remote_sums=$(ssh "$HOST" "cd $q_remote_output && sha256sum $(printf '%q ' "${FILES[@]}")" | sort)
|
|
if [[ "$local_sums" != "$remote_sums" ]]; then
|
|
print_error "Checksum mismatch after upload"
|
|
diff <(echo "$local_sums") <(echo "$remote_sums") || true
|
|
exit 1
|
|
fi
|
|
print_success "All ${#FILES[@]} package(s) verified"
|
|
echo ""
|
|
|
|
# --- publish on the host -----------------------------------------------------
|
|
|
|
print_info "Publishing on $HOST (sign -> promote -> update -> sync)..."
|
|
echo ""
|
|
if ! ssh "$HOST" "source $q_credentials && cd $q_remote_root && bin/upload-prebuilt --mirror $(printf '%q' "$MIRROR") --arch $(printf '%q' "$ARCH") --skip-prod-check"; then
|
|
print_error "Remote publish failed"
|
|
print_warning "The uploaded packages are still in $REMOTE_BUILD_OUTPUT on $HOST"
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
print_info "Published versions:"
|
|
for file in "${FILES[@]}"; do
|
|
print_step "${file%-*-*.pkg.tar.*}"
|
|
done
|
|
echo ""
|
|
print_success "Push complete!"
|