Ryan Hughes
e73b843bd2
- helpers/paths.sh: validate_mirror/require_valid_mirror for the edge|rc|stable set, and REPO_ROOT (OMARCHY_REPO_ROOT override) so a secondary checkout like the rc branch worktree publishes into the same channel tree as the primary - validate --mirror everywhere it previously accepted any string (sync-repo, promote-build, update-repo, clean-repo, remove-package) and widen the edge|stable checks in build, deploy, push-build, auto-release - build/Dockerfile: rc builds compile against rc-mirror.omarchy.org
314 lines
10 KiB
Bash
Executable File
314 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
# Push locally built packages to the repository host and publish them there.
|
|
#
|
|
# Heavy packages are quicker to build on a local machine than on the server, but
|
|
# publishing has to happen where the full repository lives: the signing key is on
|
|
# the repository host, and `bin/repo sync` can only produce a correct remote from a
|
|
# complete local tree. So this uploads the artifacts and runs the publish steps
|
|
# over ssh rather than syncing from here.
|
|
|
|
set -e
|
|
|
|
BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
|
source "$BUILD_ROOT/helpers/message-helpers.sh"
|
|
source "$BUILD_ROOT/helpers/paths.sh"
|
|
source "$BUILD_ROOT/helpers/host-helpers.sh"
|
|
|
|
HOST=""
|
|
REMOTE_ROOT="/root/omarchy-pkgs"
|
|
CREDENTIALS="/root/.omarchy/build-credentials"
|
|
PACKAGES=""
|
|
PACKAGE_FLAG_GIVEN=false
|
|
DRY_RUN=false
|
|
ASSUME_YES=false
|
|
INCLUDE_STAGED=false
|
|
|
|
print_header "Push Build to Host"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--arch)
|
|
ARCH="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--mirror)
|
|
MIRROR="$2"
|
|
if ! validate_mirror "$MIRROR"; then
|
|
print_error "Invalid mirror: $MIRROR (must be one of: $VALID_MIRRORS)"
|
|
exit 1
|
|
fi
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--package)
|
|
shift
|
|
PACKAGE_FLAG_GIVEN=true
|
|
while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do
|
|
[[ -n "$1" ]] && PACKAGES="$PACKAGES $1"
|
|
shift
|
|
done
|
|
PACKAGES="${PACKAGES# }"
|
|
;;
|
|
--host)
|
|
HOST="$2"
|
|
shift 2
|
|
;;
|
|
--remote-root)
|
|
REMOTE_ROOT="$2"
|
|
shift 2
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-y | --yes)
|
|
ASSUME_YES=true
|
|
shift
|
|
;;
|
|
--include-staged)
|
|
INCLUDE_STAGED=true
|
|
shift
|
|
;;
|
|
-h | --help)
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Upload packages from build-output/ to the repository host, then sign,"
|
|
echo "promote, update and sync them there."
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --arch <arch> Target architecture (default: x86_64)"
|
|
echo " --mirror <mirror> Mirror to publish to (edge, rc, or stable, default: edge)"
|
|
echo " --package <names> Only push these packages (space-separated)"
|
|
echo " --host <host> ssh destination (default: \$OMARCHY_REPO_HOST or .repo-host)"
|
|
echo " --remote-root <path> Repository path on the host (default: $REMOTE_ROOT)"
|
|
echo " --dry-run Show what would be pushed, transfer nothing"
|
|
echo " -y, --yes Do not ask for confirmation"
|
|
echo " --include-staged Publish packages already staged on the host too"
|
|
echo " -h, --help Show this help message"
|
|
echo ""
|
|
echo "Typical use:"
|
|
echo " bin/repo build --package nvidia-580xx-utils"
|
|
echo " bin/repo push --package nvidia-580xx-utils"
|
|
exit 0
|
|
;;
|
|
*)
|
|
print_error "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# --- host resolution ---------------------------------------------------------
|
|
|
|
if ! HOST=$(resolve_repo_host "$HOST"); then
|
|
print_no_repo_host
|
|
exit 1
|
|
fi
|
|
|
|
# --- collect artifacts -------------------------------------------------------
|
|
|
|
if [[ ! -d "$BUILD_OUTPUT_DIR" ]]; then
|
|
print_error "Build output directory not found: $BUILD_OUTPUT_DIR"
|
|
print_warning "Run bin/repo build first"
|
|
exit 1
|
|
fi
|
|
|
|
# Package files only. Signatures are produced on the host, and the repo database
|
|
# is rebuilt there, so neither should ride along.
|
|
mapfile -t ALL_FILES < <(cd "$BUILD_OUTPUT_DIR" && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig$' || true)
|
|
|
|
# "--package" with nothing after it, or with an empty variable, must not quietly
|
|
# widen to every artifact — that is the difference between shipping one package
|
|
# and shipping whatever else happens to be lying around.
|
|
if [[ "$PACKAGE_FLAG_GIVEN" == true && -z "$PACKAGES" ]]; then
|
|
print_error "--package requires at least one package name"
|
|
exit 1
|
|
fi
|
|
|
|
# On a build machine an unscoped build leaves the whole repository in
|
|
# build-output, because there is no local database to tell it what already
|
|
# exists. Interactively that is survivable — the confirmation below lists every
|
|
# package first — but with --yes nobody sees the list, so require an explicit
|
|
# selection instead.
|
|
if [[ -z "$PACKAGES" && "$ASSUME_YES" == true ]] && ! on_repo_host; then
|
|
print_error "--package is required to publish unattended from a build machine"
|
|
echo ""
|
|
echo "There is no repository database in $REPO_DIR, so a preceding unscoped"
|
|
echo "build would have rebuilt everything rather than only what changed, and"
|
|
echo "--yes would publish all ${#ALL_FILES[@]} of them without showing the list."
|
|
echo ""
|
|
echo "Name the packages to publish:"
|
|
echo " bin/repo push --package <name>"
|
|
exit 1
|
|
fi
|
|
|
|
# --package means the same thing here as it does to bin/build: a pkgbase, whose
|
|
# every output ships together. Selecting only the artifact whose filename matched
|
|
# would publish one third of a split package like nvidia-580xx-utils and silently
|
|
# leave nvidia-580xx-dkms and opencl-nvidia-580xx behind. An output's own name
|
|
# still matches, for pushing just one of them on purpose.
|
|
#
|
|
# pkgbase comes from .PKGINFO rather than the PKGBUILD: it is what makepkg
|
|
# actually recorded, and it needs no guessing about which directory built what.
|
|
pkgbase_of() {
|
|
bsdtar -xOf "$1" .PKGINFO 2>/dev/null |
|
|
awk -F ' = ' '$1 == "pkgbase" { print $2; exit }'
|
|
}
|
|
|
|
FILES=()
|
|
if [[ -z "$PACKAGES" ]]; then
|
|
FILES=("${ALL_FILES[@]}")
|
|
else
|
|
declare -A MATCHED=()
|
|
for file in "${ALL_FILES[@]}"; do
|
|
# name-version-release-arch.pkg.tar.zst -> name
|
|
pkgname="${file%-*-*-*.pkg.tar.*}"
|
|
pkgbase=$(pkgbase_of "$BUILD_OUTPUT_DIR/$file")
|
|
for wanted in $PACKAGES; do
|
|
if [[ "$pkgname" == "$wanted" || "$pkgbase" == "$wanted" ]]; then
|
|
FILES+=("$file")
|
|
MATCHED["$wanted"]=1
|
|
break
|
|
fi
|
|
done
|
|
done
|
|
|
|
for wanted in $PACKAGES; do
|
|
if [[ -z "${MATCHED[$wanted]:-}" ]]; then
|
|
print_error "No built artifact for '$wanted' in $BUILD_OUTPUT_DIR"
|
|
print_warning "Name a package or the pkgbase it was built from"
|
|
exit 1
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [[ ${#FILES[@]} -eq 0 ]]; then
|
|
print_error "No packages found in $BUILD_OUTPUT_DIR"
|
|
exit 1
|
|
fi
|
|
|
|
REMOTE_BUILD_OUTPUT="$REMOTE_ROOT/build-output/$MIRROR/$ARCH"
|
|
|
|
print_info "Host: $HOST"
|
|
print_info "Mirror: $MIRROR"
|
|
print_info "Architecture: $ARCH"
|
|
print_info "Local build output: $BUILD_OUTPUT_DIR"
|
|
print_info "Remote build output: $REMOTE_BUILD_OUTPUT"
|
|
echo ""
|
|
|
|
total=0
|
|
print_info "${#FILES[@]} package(s) to push:"
|
|
for file in "${FILES[@]}"; do
|
|
size=$(stat -c %s "$BUILD_OUTPUT_DIR/$file")
|
|
total=$((total + size))
|
|
print_step "$file ($(numfmt --to=iec --format %.1f "$size"))"
|
|
done
|
|
echo ""
|
|
print_info "Total transfer: $(numfmt --to=iec --format %.1f "$total")"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
print_warning "DRY RUN - nothing transferred"
|
|
echo ""
|
|
print_info "Would run on $HOST:"
|
|
echo " source $CREDENTIALS && cd $REMOTE_ROOT && bin/upload-prebuilt --mirror $MIRROR --arch $ARCH"
|
|
exit 0
|
|
fi
|
|
|
|
# Publishing reaches production, so confirm here. The remote publish runs
|
|
# non-interactively and cannot ask.
|
|
if [[ "$ASSUME_YES" != true ]]; then
|
|
print_warning "This publishes to PRODUCTION via $HOST ($MIRROR/$ARCH)"
|
|
read -p "Continue? (y/N) " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
print_info "Push cancelled"
|
|
exit 0
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
# --- transfer ----------------------------------------------------------------
|
|
|
|
# Remote paths are interpolated into shell command strings, so quote them for the
|
|
# remote shell rather than trusting them to contain nothing surprising.
|
|
q_remote_root=$(printf '%q' "$REMOTE_ROOT")
|
|
q_remote_output=$(printf '%q' "$REMOTE_BUILD_OUTPUT")
|
|
q_credentials=$(printf '%q' "$CREDENTIALS")
|
|
|
|
print_info "Checking host..."
|
|
if ! ssh "$HOST" "test -d $q_remote_root"; then
|
|
print_error "Repository not found on host: $REMOTE_ROOT"
|
|
print_warning "Pass --remote-root if it lives elsewhere"
|
|
exit 1
|
|
fi
|
|
ssh "$HOST" "mkdir -p $q_remote_output"
|
|
|
|
# upload-prebuilt signs and promotes everything in the host's build-output, not
|
|
# just what we are about to send. Anything already sitting there — typically the
|
|
# leftovers of an earlier failed push — would ride along unnoticed.
|
|
staged=$(ssh "$HOST" "cd $q_remote_output && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig\$' || true")
|
|
unexpected=""
|
|
if [[ -n "$staged" ]]; then
|
|
while IFS= read -r remote_file; do
|
|
[[ -z "$remote_file" ]] && continue
|
|
for file in "${FILES[@]}"; do
|
|
[[ "$remote_file" == "$file" ]] && continue 2
|
|
done
|
|
unexpected+="$remote_file"$'\n'
|
|
done <<<"$staged"
|
|
fi
|
|
|
|
if [[ -n "$unexpected" && "$INCLUDE_STAGED" != true ]]; then
|
|
print_error "The host already has staged packages this push did not build:"
|
|
echo ""
|
|
echo "$unexpected" | grep -v '^$' | sed 's/^/ /'
|
|
echo ""
|
|
echo "Publishing signs and promotes everything in $REMOTE_BUILD_OUTPUT, so these"
|
|
echo "would be published too. They are usually left over from a failed push."
|
|
echo ""
|
|
echo "Remove them on the host, or pass --include-staged to publish them as well."
|
|
exit 1
|
|
fi
|
|
print_success "Host ready"
|
|
echo ""
|
|
|
|
print_info "Uploading packages..."
|
|
# Prefix with ./ so rsync does not read an epoch's colon (spotify-1:1.2.3-...)
|
|
# as a host:path separator.
|
|
rsync_sources=()
|
|
for file in "${FILES[@]}"; do rsync_sources+=("./$file"); done
|
|
(cd "$BUILD_OUTPUT_DIR" && rsync -a --info=progress2 --partial "${rsync_sources[@]}" "$HOST:$REMOTE_BUILD_OUTPUT/")
|
|
print_success "Upload complete"
|
|
echo ""
|
|
|
|
print_info "Verifying checksums..."
|
|
local_sums=$(cd "$BUILD_OUTPUT_DIR" && sha256sum "${FILES[@]}" | sort)
|
|
remote_sums=$(ssh "$HOST" "cd $q_remote_output && sha256sum $(printf '%q ' "${FILES[@]}")" | sort)
|
|
if [[ "$local_sums" != "$remote_sums" ]]; then
|
|
print_error "Checksum mismatch after upload"
|
|
diff <(echo "$local_sums") <(echo "$remote_sums") || true
|
|
exit 1
|
|
fi
|
|
print_success "All ${#FILES[@]} package(s) verified"
|
|
echo ""
|
|
|
|
# --- publish on the host -----------------------------------------------------
|
|
|
|
print_info "Publishing on $HOST (sign -> promote -> update -> sync)..."
|
|
echo ""
|
|
if ! ssh "$HOST" "source $q_credentials && cd $q_remote_root && bin/upload-prebuilt --mirror $(printf '%q' "$MIRROR") --arch $(printf '%q' "$ARCH") --skip-prod-check"; then
|
|
print_error "Remote publish failed"
|
|
print_warning "The uploaded packages are still in $REMOTE_BUILD_OUTPUT on $HOST"
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
print_info "Published versions:"
|
|
for file in "${FILES[@]}"; do
|
|
print_step "${file%-*-*.pkg.tar.*}"
|
|
done
|
|
echo ""
|
|
print_success "Push complete!"
|