diff --git a/.gitignore b/.gitignore index 4b6c849..fce5f3c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ pkgbuilds/symfony-cli/symfony* !pkgbuilds/symfony-cli/symfony-cli.install pkgbuilds/yay/yay/ .srcdest/ -.build-host +.repo-host diff --git a/README.md b/README.md index f5eab1c..d63b0eb 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,40 @@ bin/repo update # Update database bin/repo sync # Sync to remote ``` +### Building Heavy Packages Locally + +Large packages build faster on a local machine than on the server. Build them +here, then hand the artifacts to the repository host, which signs and publishes +them: + +```bash +bin/repo deploy --package nvidia-580xx-utils # Build here, publish from the host +``` + +`deploy` is `build` followed by `push`. The two steps are also available +separately when a build needs inspecting before it ships: + +```bash +bin/repo build --package nvidia-580xx-utils # Build on the fast machine +bin/repo push --package nvidia-580xx-utils # Upload + publish on the host +``` + +`push` uploads to the host's `build-output/`, verifies checksums, and runs +`bin/upload-prebuilt` there. Do not publish from a local checkout instead: only +the repository host holds the complete repository and the signing key. + +**Name the package.** `build` asks the local repository database which packages +are already built, and a build machine has no such database, so an unscoped run +treats every package as out of date and rebuilds the whole repository. `deploy` +refuses to run unscoped when that database is missing. Unscoped builds belong on +the repository host, where `bin/repo release` does the same job against a real +database. + +Every other command in `bin/` — `sign`, `promote`, `update`, `clean`, `migrate`, +`remove`, `sync`, `release` — works on the published tree directly and is meant +to run on the repository host. `build`, `push` and `deploy` are the three that +may run elsewhere. + ## Commands ### Global Flags @@ -141,10 +175,68 @@ bin/repo sync # Sync current arch/mirror bin/repo sync --mirror stable # Sync stable bin/repo sync --arch aarch64 # Sync ARM64 bin/repo sync --skip-prod-check # No confirmation +bin/repo sync --prune # Also delete remote packages missing locally ``` Syncs package repositories to the remote server using rclone based on the configured mirror and architecture. +**Uploads are additive.** A local tree is not authoritative about what belongs on +the remote — `pkgs.omarchy.org/` is gitignored, and packages built on another +machine exist only there — so sync never deletes by default. Removing packages +from the remote requires `--prune`, which only makes sense from a complete tree. + +For the same reason sync refuses to publish a repository database built from a +tree holding fewer packages than the remote database already lists. The database +is what pacman resolves against, so a partial one hides every package it does not +know about even though the files are still on the mirror. Use `bin/repo push` to +publish packages built on another machine. + +### Deploy + +```bash +bin/repo deploy --package nvidia-580xx-utils # Build locally, publish from the host +bin/repo deploy --host root@example.com # Point at a specific repo host +bin/repo deploy --dry-run # Show the plan, change nothing +``` + +Runs `build` then `push` in one command. The repository host is resolved before +the build starts, so a missing `--host` fails immediately rather than after a long +compile. + +### Push to the Repository Host + +```bash +bin/repo push # Push everything in build-output +bin/repo push --package nvidia-580xx-utils # Push one package +bin/repo push --mirror stable --arch aarch64 # Pick mirror and architecture +bin/repo push --host root@example.com # Override the repo host +bin/repo push --dry-run # Show the plan, transfer nothing +``` + +Uploads packages from `build-output/` to the repository host and publishes them there +with `bin/upload-prebuilt` (sign → promote → update → sync). Use it when a package +is quicker to build on a local machine than on the server. + +Publishing happens on the host rather than locally for two reasons: the GPG +signing key lives there and nowhere else, and only the host holds the complete +repository that a correct database and sync require. Local machines therefore +need no secrets. + +The host comes from `--host`, `$OMARCHY_REPO_HOST`, then `.repo-host`. One +machine both serves pkgs.omarchy.org and runs the scheduled builds, so the +setting is named for the repository rather than for building, which happens +wherever you like. The same setting tells `bin/omarchy-pkgs release` which host +to poke after a release push. + +Split packages are selected by their own names, not their pkgbase — pushing +`nvidia-580xx-utils` does not carry `nvidia-580xx-dkms` along. Omit `--package` to +push everything built. + +Publishing signs and promotes everything staged on the host, not just what this +push uploaded, so `push` stops when it finds packages already staged there — +usually leftovers from a failed run. Remove them on the host, or pass +`--include-staged` to publish them too. + ### Sync AUR PKGBUILDs ```bash @@ -161,6 +253,8 @@ bin/repo migrate --arch x86_64 # Promote tested edge artifacts -> stable, bin/repo migrate --package # Promote a single package -> stable bin/repo migrate --dry-run # Preview migration and cleanup bin/repo list # List package metadata +bin/repo deploy # Build locally, then publish from the host +bin/repo push # Upload local builds to the host and publish bin/add-package # Add an AUR/local package with metadata bin/package-worktree # Create upstream/patched/current scratch workspace bin/repo remove # Remove package @@ -235,8 +329,8 @@ stable — promotion is always this explicit step. ### Build trigger After pushing, the command triggers the build host over ssh when -`OMARCHY_BUILD_HOST` is set (env var, or a hostname in the git-ignored -`.build-host` file). Without it, the 6-hourly auto-release timer picks up the +`OMARCHY_REPO_HOST` is set (env var, or a hostname in the git-ignored +`.repo-host` file). Without it, the 6-hourly auto-release timer picks up the change on its own. ## Directory Structure diff --git a/bin/deploy b/bin/deploy new file mode 100755 index 0000000..cfcf171 --- /dev/null +++ b/bin/deploy @@ -0,0 +1,151 @@ +#!/bin/bash +# Build packages locally, then publish them from the repository host. +# +# The two halves of shipping a package built on a local machine: bin/build +# produces the artifacts, bin/push-build hands them to the host that owns the +# signing key and the complete repository. + +set -e + +SCRIPT_DIR=$(realpath "${BASH_SOURCE[0]%/*}") +BUILD_ROOT=$(realpath "$SCRIPT_DIR/..") +source "$BUILD_ROOT/helpers/message-helpers.sh" +source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/host-helpers.sh" + +PACKAGES=() +PACKAGE_FLAG_GIVEN=false +HOST="" +REMOTE_ROOT="" +DRY_RUN=false +ASSUME_YES=false + +print_header "Deploy (build + push)" + +while [[ $# -gt 0 ]]; do + case $1 in + --arch) + ARCH="$2" + update_arch_paths + shift 2 + ;; + --mirror) + MIRROR="$2" + if [[ "$MIRROR" != "edge" && "$MIRROR" != "stable" ]]; then + print_error "Invalid mirror: $MIRROR (must be 'edge' or 'stable')" + exit 1 + fi + update_arch_paths + shift 2 + ;; + --package) + shift + PACKAGE_FLAG_GIVEN=true + while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do + [[ -n "$1" ]] && PACKAGES+=("$1") + shift + done + ;; + --host) + HOST="$2" + shift 2 + ;; + --remote-root) + REMOTE_ROOT="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -y | --yes) + ASSUME_YES=true + shift + ;; + -h | --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Build packages here, then publish them from the repository host." + echo "" + echo "Options:" + echo " --arch Target architecture (default: x86_64)" + echo " --mirror Mirror to publish to (edge or stable, default: edge)" + echo " --package Build and push only these packages (space-separated)" + echo " --host ssh destination (default: \$OMARCHY_REPO_HOST or .repo-host)" + echo " --remote-root Repository path on the host (default: /root/omarchy-pkgs)" + echo " --dry-run Show the plan, build nothing and transfer nothing" + echo " -y, --yes Do not ask for confirmation before publishing" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " $0 --package nvidia-580xx-utils" + echo " $0 --package nvidia-580xx-utils --host root@example.com" + exit 0 + ;; + *) + print_error "Unknown option: $1" + exit 1 + ;; + esac +done + +if [[ "$PACKAGE_FLAG_GIVEN" == true && ${#PACKAGES[@]} -eq 0 ]]; then + print_error "--package requires at least one package name" + exit 1 +fi + +# bin/build asks the local repository database what is already built. On a build +# machine that database does not exist, so every package looks unbuilt and an +# unscoped run rebuilds the entire repository and publishes it. Deploying from +# here is for the occasional heavy package, so name it. +if [[ ${#PACKAGES[@]} -eq 0 ]] && ! on_repo_host; then + print_error "--package is required when deploying from a build machine" + echo "" + echo "There is no repository database in:" + echo " $REPO_DIR" + echo "" + echo "bin/build uses it to tell which packages are already built, so without it" + echo "every package looks out of date and this would build and publish all of" + echo "them. Name the package you came here to build:" + echo " bin/repo deploy --package " + echo "" + echo "Unscoped builds belong on the repository host, where 'bin/repo release'" + echo "does the same job against a real database." + exit 1 +fi + +echo "" +print_info "This will:" +echo " 1. Build packages locally" +echo " 2. Upload them to the repository host" +echo " 3. Sign, promote, update and sync them there" +echo "" + +BUILD_ARGS=("--arch" "$ARCH" "--mirror" "$MIRROR") +PUSH_ARGS=("--arch" "$ARCH" "--mirror" "$MIRROR") + +if [[ ${#PACKAGES[@]} -gt 0 ]]; then + BUILD_ARGS+=("--package" "${PACKAGES[@]}") + PUSH_ARGS+=("--package" "${PACKAGES[@]}") +fi +[[ -n "$HOST" ]] && PUSH_ARGS+=("--host" "$HOST") +[[ -n "$REMOTE_ROOT" ]] && PUSH_ARGS+=("--remote-root" "$REMOTE_ROOT") +[[ "$DRY_RUN" == true ]] && BUILD_ARGS+=("--dry-run") && PUSH_ARGS+=("--dry-run") +[[ "$ASSUME_YES" == true ]] && PUSH_ARGS+=("--yes") + +# Resolve the repository host before spending build time on packages that cannot ship. +if [[ "$DRY_RUN" != true ]]; then + if ! resolve_repo_host "$HOST" >/dev/null; then + print_no_repo_host + exit 1 + fi +fi + +print_info "Step 1/2: Building..." +echo "" +"$SCRIPT_DIR/build" "${BUILD_ARGS[@]}" +echo "" + +print_info "Step 2/2: Pushing to the repository host..." +echo "" +"$SCRIPT_DIR/push-build" "${PUSH_ARGS[@]}" diff --git a/bin/omarchy-pkgs b/bin/omarchy-pkgs index 2b151df..0c3be82 100755 --- a/bin/omarchy-pkgs +++ b/bin/omarchy-pkgs @@ -15,6 +15,7 @@ set -e BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") source "$BUILD_ROOT/helpers/message-helpers.sh" +source "$BUILD_ROOT/helpers/host-helpers.sh" UPSTREAM_URL="${OMARCHY_UPSTREAM_URL:-https://github.com/basecamp/omarchy.git}" EDGE_DB_URL="${OMARCHY_EDGE_DB_URL:-https://pkgs.omarchy.org/edge/x86_64/omarchy.db.tar.zst}" @@ -43,6 +44,8 @@ Options for release: --commit (rc) Upstream commit to pin (default: tip of --ref) --ref (rc) Upstream branch whose tip to pin (default: $DEFAULT_RC_REF) --yes Skip confirmation prompts + --host ssh destination of the repository host to trigger, + overriding \$OMARCHY_REPO_HOST and .repo-host --no-push Rewrite and commit locally; skip push and build trigger --pr When releasing from a non-master branch, open a GitHub PR to master with gh after pushing @@ -312,10 +315,9 @@ regenerate_checksums() { # --- trigger ----------------------------------------------------------------- trigger_build_host() { - local host="${OMARCHY_BUILD_HOST:-}" - [[ -z "$host" && -f "$BUILD_ROOT/.build-host" ]] && host=$(<"$BUILD_ROOT/.build-host") - if [[ -z "$host" ]]; then - print_info "No build host configured (set OMARCHY_BUILD_HOST or $BUILD_ROOT/.build-host)." + local host + if ! host=$(resolve_repo_host "${REPO_HOST_OVERRIDE:-}"); then + print_info "No repository host configured (set OMARCHY_REPO_HOST or $BUILD_ROOT/.repo-host)." print_info "The 6-hourly auto-release timer will pick this up, or trigger manually:" echo " ssh 'git -C /root/omarchy-pkgs pull --ff-only && touch /root/.state/.sync-needed-edge && systemctl start omarchy-auto-release-edge.service'" return 0 @@ -341,6 +343,7 @@ cmd_release() { --force) force=true; shift ;; --commit) commit_arg="$2"; shift 2 ;; --ref) ref="$2"; shift 2 ;; + --host) REPO_HOST_OVERRIDE="$2"; shift 2 ;; --yes) assume_yes=true; shift ;; --dry-run) dry_run=true; shift ;; -h | --help) show_usage; exit 0 ;; diff --git a/bin/push-build b/bin/push-build new file mode 100755 index 0000000..21523e8 --- /dev/null +++ b/bin/push-build @@ -0,0 +1,301 @@ +#!/bin/bash +# Push locally built packages to the repository host and publish them there. +# +# Heavy packages are quicker to build on a local machine than on the server, but +# publishing has to happen where the full repository lives: the signing key is on +# the repository host, and `bin/repo sync` can only produce a correct remote from a +# complete local tree. So this uploads the artifacts and runs the publish steps +# over ssh rather than syncing from here. + +set -e + +BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") +source "$BUILD_ROOT/helpers/message-helpers.sh" +source "$BUILD_ROOT/helpers/paths.sh" +source "$BUILD_ROOT/helpers/host-helpers.sh" + +HOST="" +REMOTE_ROOT="/root/omarchy-pkgs" +CREDENTIALS="/root/.omarchy/build-credentials" +PACKAGES="" +PACKAGE_FLAG_GIVEN=false +DRY_RUN=false +ASSUME_YES=false +INCLUDE_STAGED=false + +print_header "Push Build to Host" + +while [[ $# -gt 0 ]]; do + case $1 in + --arch) + ARCH="$2" + update_arch_paths + shift 2 + ;; + --mirror) + MIRROR="$2" + if [[ "$MIRROR" != "edge" && "$MIRROR" != "stable" ]]; then + print_error "Invalid mirror: $MIRROR (must be 'edge' or 'stable')" + exit 1 + fi + update_arch_paths + shift 2 + ;; + --package) + shift + PACKAGE_FLAG_GIVEN=true + while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do + [[ -n "$1" ]] && PACKAGES="$PACKAGES $1" + shift + done + PACKAGES="${PACKAGES# }" + ;; + --host) + HOST="$2" + shift 2 + ;; + --remote-root) + REMOTE_ROOT="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -y | --yes) + ASSUME_YES=true + shift + ;; + --include-staged) + INCLUDE_STAGED=true + shift + ;; + -h | --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Upload packages from build-output/ to the repository host, then sign," + echo "promote, update and sync them there." + echo "" + echo "Options:" + echo " --arch Target architecture (default: x86_64)" + echo " --mirror Mirror to publish to (edge or stable, default: edge)" + echo " --package Only push these packages (space-separated)" + echo " --host ssh destination (default: \$OMARCHY_REPO_HOST or .repo-host)" + echo " --remote-root Repository path on the host (default: $REMOTE_ROOT)" + echo " --dry-run Show what would be pushed, transfer nothing" + echo " -y, --yes Do not ask for confirmation" + echo " --include-staged Publish packages already staged on the host too" + echo " -h, --help Show this help message" + echo "" + echo "Typical use:" + echo " bin/repo build --package nvidia-580xx-utils" + echo " bin/repo push --package nvidia-580xx-utils" + exit 0 + ;; + *) + print_error "Unknown option: $1" + exit 1 + ;; + esac +done + +# --- host resolution --------------------------------------------------------- + +if ! HOST=$(resolve_repo_host "$HOST"); then + print_no_repo_host + exit 1 +fi + +# --- collect artifacts ------------------------------------------------------- + +if [[ ! -d "$BUILD_OUTPUT_DIR" ]]; then + print_error "Build output directory not found: $BUILD_OUTPUT_DIR" + print_warning "Run bin/repo build first" + exit 1 +fi + +# Package files only. Signatures are produced on the host, and the repo database +# is rebuilt there, so neither should ride along. +mapfile -t ALL_FILES < <(cd "$BUILD_OUTPUT_DIR" && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig$' || true) + +# "--package" with nothing after it, or with an empty variable, must not quietly +# widen to every artifact — that is the difference between shipping one package +# and shipping whatever else happens to be lying around. +if [[ "$PACKAGE_FLAG_GIVEN" == true && -z "$PACKAGES" ]]; then + print_error "--package requires at least one package name" + exit 1 +fi + +# On a build machine an unscoped build leaves the whole repository in +# build-output, because there is no local database to tell it what already +# exists. Interactively that is survivable — the confirmation below lists every +# package first — but with --yes nobody sees the list, so require an explicit +# selection instead. +if [[ -z "$PACKAGES" && "$ASSUME_YES" == true ]] && ! on_repo_host; then + print_error "--package is required to publish unattended from a build machine" + echo "" + echo "There is no repository database in $REPO_DIR, so a preceding unscoped" + echo "build would have rebuilt everything rather than only what changed, and" + echo "--yes would publish all ${#ALL_FILES[@]} of them without showing the list." + echo "" + echo "Name the packages to publish:" + echo " bin/repo push --package " + exit 1 +fi + +FILES=() +if [[ -z "$PACKAGES" ]]; then + FILES=("${ALL_FILES[@]}") +else + for file in "${ALL_FILES[@]}"; do + # name-version-release-arch.pkg.tar.zst -> name + pkgname="${file%-*-*-*.pkg.tar.*}" + for wanted in $PACKAGES; do + if [[ "$pkgname" == "$wanted" ]]; then + FILES+=("$file") + break + fi + done + done + + for wanted in $PACKAGES; do + found=false + for file in "${FILES[@]}"; do + [[ "${file%-*-*-*.pkg.tar.*}" == "$wanted" ]] && found=true && break + done + if [[ "$found" != true ]]; then + print_error "No built artifact for '$wanted' in $BUILD_OUTPUT_DIR" + print_warning "Split packages are named after their outputs, not their pkgbase" + exit 1 + fi + done +fi + +if [[ ${#FILES[@]} -eq 0 ]]; then + print_error "No packages found in $BUILD_OUTPUT_DIR" + exit 1 +fi + +REMOTE_BUILD_OUTPUT="$REMOTE_ROOT/build-output/$MIRROR/$ARCH" + +print_info "Host: $HOST" +print_info "Mirror: $MIRROR" +print_info "Architecture: $ARCH" +print_info "Local build output: $BUILD_OUTPUT_DIR" +print_info "Remote build output: $REMOTE_BUILD_OUTPUT" +echo "" + +total=0 +print_info "${#FILES[@]} package(s) to push:" +for file in "${FILES[@]}"; do + size=$(stat -c %s "$BUILD_OUTPUT_DIR/$file") + total=$((total + size)) + print_step "$file ($(numfmt --to=iec --format %.1f "$size"))" +done +echo "" +print_info "Total transfer: $(numfmt --to=iec --format %.1f "$total")" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + print_warning "DRY RUN - nothing transferred" + echo "" + print_info "Would run on $HOST:" + echo " source $CREDENTIALS && cd $REMOTE_ROOT && bin/upload-prebuilt --mirror $MIRROR --arch $ARCH" + exit 0 +fi + +# Publishing reaches production, so confirm here. The remote publish runs +# non-interactively and cannot ask. +if [[ "$ASSUME_YES" != true ]]; then + print_warning "This publishes to PRODUCTION via $HOST ($MIRROR/$ARCH)" + read -p "Continue? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + print_info "Push cancelled" + exit 0 + fi + echo +fi + +# --- transfer ---------------------------------------------------------------- + +# Remote paths are interpolated into shell command strings, so quote them for the +# remote shell rather than trusting them to contain nothing surprising. +q_remote_root=$(printf '%q' "$REMOTE_ROOT") +q_remote_output=$(printf '%q' "$REMOTE_BUILD_OUTPUT") +q_credentials=$(printf '%q' "$CREDENTIALS") + +print_info "Checking host..." +if ! ssh "$HOST" "test -d $q_remote_root"; then + print_error "Repository not found on host: $REMOTE_ROOT" + print_warning "Pass --remote-root if it lives elsewhere" + exit 1 +fi +ssh "$HOST" "mkdir -p $q_remote_output" + +# upload-prebuilt signs and promotes everything in the host's build-output, not +# just what we are about to send. Anything already sitting there — typically the +# leftovers of an earlier failed push — would ride along unnoticed. +staged=$(ssh "$HOST" "cd $q_remote_output && ls -1 *.pkg.tar.* 2>/dev/null | grep -v '\.sig\$' || true") +unexpected="" +if [[ -n "$staged" ]]; then + while IFS= read -r remote_file; do + [[ -z "$remote_file" ]] && continue + for file in "${FILES[@]}"; do + [[ "$remote_file" == "$file" ]] && continue 2 + done + unexpected+="$remote_file"$'\n' + done <<<"$staged" +fi + +if [[ -n "$unexpected" && "$INCLUDE_STAGED" != true ]]; then + print_error "The host already has staged packages this push did not build:" + echo "" + echo "$unexpected" | grep -v '^$' | sed 's/^/ /' + echo "" + echo "Publishing signs and promotes everything in $REMOTE_BUILD_OUTPUT, so these" + echo "would be published too. They are usually left over from a failed push." + echo "" + echo "Remove them on the host, or pass --include-staged to publish them as well." + exit 1 +fi +print_success "Host ready" +echo "" + +print_info "Uploading packages..." +# Prefix with ./ so rsync does not read an epoch's colon (spotify-1:1.2.3-...) +# as a host:path separator. +rsync_sources=() +for file in "${FILES[@]}"; do rsync_sources+=("./$file"); done +(cd "$BUILD_OUTPUT_DIR" && rsync -a --info=progress2 --partial "${rsync_sources[@]}" "$HOST:$REMOTE_BUILD_OUTPUT/") +print_success "Upload complete" +echo "" + +print_info "Verifying checksums..." +local_sums=$(cd "$BUILD_OUTPUT_DIR" && sha256sum "${FILES[@]}" | sort) +remote_sums=$(ssh "$HOST" "cd $q_remote_output && sha256sum $(printf '%q ' "${FILES[@]}")" | sort) +if [[ "$local_sums" != "$remote_sums" ]]; then + print_error "Checksum mismatch after upload" + diff <(echo "$local_sums") <(echo "$remote_sums") || true + exit 1 +fi +print_success "All ${#FILES[@]} package(s) verified" +echo "" + +# --- publish on the host ----------------------------------------------------- + +print_info "Publishing on $HOST (sign -> promote -> update -> sync)..." +echo "" +if ! ssh "$HOST" "source $q_credentials && cd $q_remote_root && bin/upload-prebuilt --mirror $(printf '%q' "$MIRROR") --arch $(printf '%q' "$ARCH") --skip-prod-check"; then + print_error "Remote publish failed" + print_warning "The uploaded packages are still in $REMOTE_BUILD_OUTPUT on $HOST" + exit 1 +fi +echo "" + +print_info "Published versions:" +for file in "${FILES[@]}"; do + print_step "${file%-*-*.pkg.tar.*}" +done +echo "" +print_success "Push complete!" diff --git a/bin/repo b/bin/repo index be33824..0e2c40e 100755 --- a/bin/repo +++ b/bin/repo @@ -58,6 +58,8 @@ show_usage() { echo " list List source package metadata (use --repo for published repo)" echo " remove Remove a specific package" echo " sync Sync repository to remote" + echo " push Upload local builds to the repository host and publish them there" + echo " deploy Build locally, then push: one command from a build machine" echo "" echo "Typical workflows:" echo " $0 release # Complete release workflow" @@ -125,6 +127,14 @@ sync) "$SCRIPT_DIR/sync-repo" "$@" 2>&1 | tee "$LOG_FILE" exit ${PIPESTATUS[0]} ;; +push) + "$SCRIPT_DIR/push-build" "$@" 2>&1 | tee "$LOG_FILE" + exit ${PIPESTATUS[0]} + ;; +deploy) + "$SCRIPT_DIR/deploy" "$@" 2>&1 | tee "$LOG_FILE" + exit ${PIPESTATUS[0]} + ;; -h | --help | help) show_usage ;; diff --git a/bin/sync-repo b/bin/sync-repo index e766828..fc8394f 100755 --- a/bin/sync-repo +++ b/bin/sync-repo @@ -9,24 +9,37 @@ source "$BUILD_ROOT/helpers/paths.sh" DEFAULT_REMOTE="pkgs.omarchy.org:omarchy-pkgs" REMOTE="$DEFAULT_REMOTE" SKIP_PROD_CHECK=false +PRUNE=false # Print header print_header "Sync Repository to Remote" +# This script has no `set -e`, so a `shift 2` past the end of the argument list +# fails without consuming anything and the loop spins forever. Check first. +require_value() { + if [[ $# -lt 2 || -z "$2" ]]; then + print_error "Option $1 requires a value" + exit 1 + fi +} + # Parse arguments while [[ $# -gt 0 ]]; do case $1 in --arch) + require_value "$@" ARCH="$2" update_arch_paths shift 2 ;; --mirror) + require_value "$@" MIRROR="$2" update_arch_paths shift 2 ;; --remote) + require_value "$@" REMOTE="$2" shift 2 ;; @@ -34,6 +47,10 @@ while [[ $# -gt 0 ]]; do SKIP_PROD_CHECK=true shift ;; + --prune) + PRUNE=true + shift + ;; -h | --help) echo "Usage: $0 [OPTIONS]" echo "" @@ -42,7 +59,11 @@ while [[ $# -gt 0 ]]; do echo " --mirror Mirror to use (edge or stable, default: edge)" echo " --remote Rclone remote destination (default: $DEFAULT_REMOTE)" echo " --skip-prod-check Skip production sync confirmation" + echo " --prune Also delete remote packages missing locally" echo " -h, --help Show this help message" + echo "" + echo "Uploads are additive by default. Removing packages from the remote" + echo "requires --prune, which only makes sense from a complete local tree." exit 0 ;; *) @@ -80,21 +101,133 @@ fi print_info "Syncing to: $REMOTE/$DESTINATION_DIRECTORY" -# First sync packages (excluding database files to ensure packages are uploaded first) -# Use --ignore-existing to not overwrite different versions already on remote -print_info "Syncing packages..." -rclone sync "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \ - --s3-no-head \ - --exclude "omarchy.db*" \ - --exclude "omarchy.files*" \ - --ignore-existing \ - --copy-links --delete-after -v +# The database is what users actually resolve against, and repo-add builds it +# from this tree alone. Publishing one built from a partial tree hides every +# package it does not know about, even though the files are still on the remote. +# Refuse to shrink the package list unless that is the stated intent. +# +# Compare package names, not file counts: this tree keeps several versions of +# each package (bin/repo clean --keep 2) while the database carries one entry per +# name, so counting files would compare unrelated quantities and let a partial +# tree through whenever its spare versions made up the difference. +strip_version() { sed -E 's/-[^-]+-[^-]+-[^-]+\.pkg\.tar\.[^.]+$//'; } + +LOCAL_NAMES=$(ls -1 "$REPO_DIR" 2>/dev/null | grep -v '\.sig$' | grep '\.pkg\.tar\.' | + strip_version | sort -u) + +# Distinguish "no repository there yet" from "cannot read the repository". Only +# the first is safe to treat as an empty remote; failing open on a credential or +# network error is how a partial database reaches production. +REMOTE_LISTING=$(rclone lsf "$REMOTE/$DESTINATION_DIRECTORY/" --s3-no-head 2>&1) +RCLONE_STATUS=$? + +# rclone exit 3 is "directory not found", which is what a mirror that has never +# been published looks like. Every other failure means the remote could not be +# read, and an unread remote must not be mistaken for an empty one. +if [[ $RCLONE_STATUS -eq 3 ]]; then + REMOTE_LISTING="" +elif [[ $RCLONE_STATUS -ne 0 ]]; then + print_error "Cannot read the remote repository (rclone exit $RCLONE_STATUS)" + echo "$REMOTE_LISTING" + echo "" + echo "Refusing to sync: an unreadable remote cannot be checked for packages" + echo "this tree would hide." + exit 1 +fi + +if grep -qx 'omarchy\.db' <<<"$REMOTE_LISTING"; then + # bsdtar, not tar: the database is compressed and GNU tar will not detect that + # on a pipe. repo-add has used both gzip and zstd, so let libarchive decide. + REMOTE_DB_FILE=$(mktemp) + trap 'rm -f "$REMOTE_DB_FILE"' EXIT + rclone cat "$REMOTE/$DESTINATION_DIRECTORY/omarchy.db" --s3-no-head >"$REMOTE_DB_FILE" 2>/dev/null + REMOTE_NAMES=$(bsdtar -tf "$REMOTE_DB_FILE" 2>/dev/null | sed 's|/.*||' | + sed -E 's/-[^-]+-[^-]+$//' | sort -u) + + if [[ -z "$REMOTE_NAMES" ]]; then + print_error "The remote database exists but could not be read" + echo "" + echo "Refusing to sync rather than assume the remote is empty. Check that" + echo "bsdtar is installed and that omarchy.db is not corrupt." + exit 1 + fi + + HIDDEN=$(comm -23 <(echo "$REMOTE_NAMES") <(echo "$LOCAL_NAMES")) + HIDDEN_COUNT=$(grep -c '' <<<"$HIDDEN") + [[ -z "$HIDDEN" ]] && HIDDEN_COUNT=0 + + if [[ "$HIDDEN_COUNT" -gt 0 && "$PRUNE" != true ]]; then + print_error "$HIDDEN_COUNT package(s) in the remote database are missing from this tree" + echo "" + echo "$HIDDEN" | head -10 | sed 's/^/ /' + [[ "$HIDDEN_COUNT" -gt 10 ]] && echo " ... and $((HIDDEN_COUNT - 10)) more" + echo "" + echo "Publishing a database built here would hide them, even though their" + echo "files remain on the mirror." + echo "" + echo "To publish packages built on this machine, push them to the build host," + echo "which holds the complete repository:" + echo " bin/repo push --mirror $MIRROR --arch $ARCH" + echo "" + echo "If shrinking the repository is genuinely what you want, pass --prune." + exit 1 + fi +fi + +# Upload packages first, database last, so the remote never advertises a package +# it does not yet have. +# +# This is `copy`, not `sync`: a local tree is not authoritative about what should +# exist on the remote. Packages built on another machine live only in that +# machine's build-output, and pkgs.omarchy.org/ is gitignored, so any checkout +# that has not run a full release is missing nearly everything. `sync` would read +# those absences as deletions and empty the repository. --ignore-existing keeps +# versions already published from being overwritten. +if [[ "$PRUNE" == true ]]; then + print_warning "Pruning: remote packages missing from $REPO_DIR will be DELETED" + if [[ "$SKIP_PROD_CHECK" != true ]]; then + read -p "Prune the remote to match this tree? (y/N) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + print_info "Sync cancelled" + exit 0 + fi + fi + print_info "Syncing packages (with prune)..." + if ! rclone sync "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \ + --s3-no-head \ + --exclude "omarchy.db*" \ + --exclude "omarchy.files*" \ + --ignore-existing \ + --copy-links --delete-after -v; then + print_error "Package sync failed — not publishing the database" + exit 1 + fi +else + print_info "Syncing packages..." + if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \ + --s3-no-head \ + --exclude "omarchy.db*" \ + --exclude "omarchy.files*" \ + --ignore-existing \ + --copy-links -v; then + print_error "Package upload failed — not publishing the database" + print_warning "The remote database still describes the previous contents, so" + print_warning "the repository is unchanged and consistent." + exit 1 + fi +fi # Then sync database files last to ensure repository integrity print_info "Updating repository database..." -rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \ +if ! rclone copy "$REPO_DIR" "$REMOTE/$DESTINATION_DIRECTORY" \ --s3-no-head \ --include "omarchy.*" \ - --checksum --copy-links -v + --checksum --copy-links -v; then + print_error "Database upload failed" + print_warning "Packages were uploaded but the database still describes the" + print_warning "previous contents. Re-run sync to finish publishing them." + exit 1 +fi print_success "Sync complete!" diff --git a/bin/upload-prebuilt b/bin/upload-prebuilt index 0bd3f4a..d4b4831 100755 --- a/bin/upload-prebuilt +++ b/bin/upload-prebuilt @@ -5,7 +5,28 @@ set -e SCRIPT_DIR=$(realpath "${BASH_SOURCE[0]%/*}") -"$SCRIPT_DIR/repo" sign "$@" -"$SCRIPT_DIR/repo" promote "$@" -"$SCRIPT_DIR/repo" update "$@" -"$SCRIPT_DIR/repo" sync "$@" +# Only sync understands the publishing flags; sign, promote and update reject +# unknown options outright, so they cannot be forwarded blindly. +COMMON_ARGS=() +SYNC_ARGS=() +while [[ $# -gt 0 ]]; do + case $1 in + --skip-prod-check | --prune) + SYNC_ARGS+=("$1") + shift + ;; + --remote) + SYNC_ARGS+=("$1" "$2") + shift 2 + ;; + *) + COMMON_ARGS+=("$1") + shift + ;; + esac +done + +"$SCRIPT_DIR/repo" sign "${COMMON_ARGS[@]}" +"$SCRIPT_DIR/repo" promote "${COMMON_ARGS[@]}" +"$SCRIPT_DIR/repo" update "${COMMON_ARGS[@]}" +"$SCRIPT_DIR/repo" sync "${COMMON_ARGS[@]}" "${SYNC_ARGS[@]}" diff --git a/helpers/host-helpers.sh b/helpers/host-helpers.sh new file mode 100644 index 0000000..9f693d6 --- /dev/null +++ b/helpers/host-helpers.sh @@ -0,0 +1,52 @@ +# Resolving the Omarchy repository host +# +# One machine both serves pkgs.omarchy.org and runs the scheduled builds. The +# commands that reach it are doing repository work — uploading artifacts, +# signing, publishing — so the setting is named for the repository rather than +# for building, which happens on whatever machine the operator prefers. + +# Usage: resolve_repo_host [explicit-host] +# Prints the host, or nothing when none is configured. +resolve_repo_host() { + local explicit="${1:-}" + + if [[ -n "$explicit" ]]; then + echo "$explicit" + return 0 + fi + + if [[ -n "${OMARCHY_REPO_HOST:-}" ]]; then + echo "$OMARCHY_REPO_HOST" + return 0 + fi + + if [[ -f "$BUILD_ROOT/.repo-host" ]]; then + # Ignore blank lines and comments so the file can be annotated. + local value + value=$(grep -vE '^\s*(#|$)' "$BUILD_ROOT/.repo-host" | head -1 | tr -d '[:space:]') + if [[ -n "$value" ]]; then + echo "$value" + return 0 + fi + fi + + return 1 +} + +# True when this checkout holds the published repository, which in practice means +# this machine is the repository host. Every other command in bin/ works on that +# tree directly; build, push and deploy are the ones that may run elsewhere. +# +# The database is the marker rather than the directory: bin/build creates empty +# mirror directories as a side effect, so their presence proves nothing. +on_repo_host() { + [[ -f "$REPO_DIR/omarchy.db" || -f "$REPO_DIR/omarchy.db.tar.zst" ]] +} + +# Shared wording so every command explains configuration the same way. +print_no_repo_host() { + print_error "No repository host configured" + echo "" + echo "Pass --host, set OMARCHY_REPO_HOST, or write the destination to:" + echo " $BUILD_ROOT/.repo-host" +}