diff --git a/README.md b/README.md index 887f7bf..3694f16 100644 --- a/README.md +++ b/README.md @@ -756,6 +756,31 @@ That cadence is only safe because of three guards: `bin/repo timers` reports all of this: schedules, last results, what is queued, what is failing and when it will retry, and whether the lock is held. +### Build reports + +Every release run reports to Basecamp Campfire β€” not just failures, so a push +can be followed all the way to the mirror without watching the host: + +| Event | Message | +|---|---| +| πŸ”¨ Started | channel, arch, host, and the commit being built | +| βœ… Published | the packages and versions that went out, duration, and the channel URL | +| πŸ“¦ Promoted | what `advance` moved between channels (including the rc bootstrap and fast-ring replication) | +| πŸ“¦ No changes | the run found nothing to build | +| πŸ”΄ Failed | which step failed, the commit, and the last 25 log lines | + +Release traffic gets its own chat so it does not drown the repository chat +that the AUR/upstream sync workflows post to. Export the destination in +`/root/.omarchy/build-credentials`: + +```bash +export OMARCHY_RELEASE_CHATBOT_URL="https://3.basecamp.com//integrations//buckets//chats//lines" +``` + +Without it, release reports fall back to `BASECAMP_CHATBOT_URL`; with neither +set they are silently skipped. `bin/setup` reports which of the three cases +applies. + Check on all of it with `bin/repo timers` β€” schedule, each unit's last run and whether it succeeded, what is queued, whether a release is running right now, and any failed units. Like the other host commands it forwards over ssh, so diff --git a/bin/advance-channel b/bin/advance-channel index 125b4a4..3aae88e 100755 --- a/bin/advance-channel +++ b/bin/advance-channel @@ -15,6 +15,7 @@ source "$BUILD_ROOT/helpers/message-helpers.sh" source "$BUILD_ROOT/helpers/paths.sh" source "$BUILD_ROOT/helpers/lock-helpers.sh" source "$BUILD_ROOT/helpers/package-metadata.sh" +source "$BUILD_ROOT/helpers/basecamp-notifier.sh" FROM="" TO="" @@ -223,6 +224,7 @@ package_eligible() { mkdir -p "$TARGET_DIR" COPIED=0 +COPIED_FILES="" PRESENT=0 SKIPPED=0 MISSING_FILES=() @@ -277,6 +279,7 @@ while IFS=$'\t' read -r name base filename; do echo " copied: $filename (+ .sig)" fi COPIED=$((COPIED + 1)) + COPIED_FILES+="$filename"$'\n' done < <(read_manifest) echo "" @@ -330,3 +333,23 @@ SYNC_ARGS=("--mirror" "$TO" "--arch" "$ARCH") } print_success "Advance complete: $FROM -> $TO" + +# A promotion is how something reaches users, so say what moved and where. +if ((COPIED > 0)); then + moved="" + shown=0 + while IFS= read -r moved_file; do + [[ -z "$moved_file" ]] && continue + ((shown >= 25)) && break + moved+="
β€’ $(package_file_label "$moved_file" | basecamp_html_escape)" + shown=$((shown + 1)) + done <<<"$COPIED_FILES" + ((COPIED > shown)) && moved+="
β€’ …and $((COPIED - shown)) more" + + label="Promoted $FROM β†’ $TO" + [[ "$BOOTSTRAP" == true ]] && label="Bootstrapped the $TO channel from $FROM" + [[ "$FAST_RING_ONLY" == true ]] && label="Replicated fast-ring $FROM β†’ $TO" + + notify_info "$label" \ + "Arch: $ARCH Β· $(hostname -s 2>/dev/null || echo host)
$COPIED package(s) moved:$moved

Live at https://pkgs.omarchy.org/$TO/$ARCH/" +fi diff --git a/bin/release b/bin/release index 6e6d41e..813f55f 100755 --- a/bin/release +++ b/bin/release @@ -92,6 +92,24 @@ if [[ "$DRY_RUN" != true ]]; then acquire_release_lock || exit 1 fi +# --- reporting --------------------------------------------------------------- + +RELEASE_STARTED_AT=$(date +%s) +RELEASE_COMMIT=$(git -C "$BUILD_ROOT" rev-parse --short HEAD 2>/dev/null || echo "unknown") +RELEASE_COMMIT_SUBJECT=$(git -C "$BUILD_ROOT" log -1 --pretty=%s 2>/dev/null || echo "") +RELEASE_CONTEXT="Channel: $MIRROR Β· Arch: $ARCH Β· $(hostname -s 2>/dev/null || echo host)" +if [[ "$RELEASE_COMMIT" != "unknown" ]]; then + RELEASE_CONTEXT+="
Commit: $RELEASE_COMMIT $(basecamp_html_escape <<<"$RELEASE_COMMIT_SUBJECT")" +fi + +# Captured after the build step: promote MOVES these files out of build-output, +# so the list has to be taken while they are still there. +BUILT_FILES="" + +if [[ "$DRY_RUN" != true ]]; then + notify_start "Release started: $MIRROR" "$RELEASE_CONTEXT" +fi + # Step 1: Build echo "" if [[ "$DRY_RUN" == true ]]; then @@ -101,7 +119,7 @@ else fi "$BUILD_ROOT/bin/build" "${BUILD_ARGS[@]}" || { print_error "Build failed" - notify_error "Release failed: Build step failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: Build step failed" "$RELEASE_CONTEXT" exit 1 } @@ -111,12 +129,17 @@ if [[ "$DRY_RUN" == true ]]; then exit 0 fi +BUILT_FILES=$(built_package_files "$BUILD_OUTPUT_DIR") +BUILT_COUNT=$(grep -c '' <<<"$BUILT_FILES") +[[ -z "$BUILT_FILES" ]] && BUILT_COUNT=0 +print_info "Built $BUILT_COUNT package(s) this run" + # Step 2: Sign echo "" print_info "Step 2/6: Signing packages..." "$BUILD_ROOT/bin/sign" --arch "$ARCH" --mirror "$MIRROR" || { print_error "Signing failed" - notify_error "Release failed: Signing step failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: Signing step failed" "$RELEASE_CONTEXT" exit 1 } @@ -125,7 +148,7 @@ echo "" print_info "Step 3/6: Promoting to production..." "$BUILD_ROOT/bin/promote-build" --arch "$ARCH" --mirror "$MIRROR" || { print_error "Promotion failed" - notify_error "Release failed: Promotion step failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: Promotion step failed" "$RELEASE_CONTEXT" exit 1 } @@ -134,7 +157,7 @@ echo "" print_info "Step 4/6: Cleaning old versions..." "$BUILD_ROOT/bin/clean-repo" --arch "$ARCH" --mirror "$MIRROR" || { print_error "Cleaning failed" - notify_error "Release failed: Clean step failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: Clean step failed" "$RELEASE_CONTEXT" exit 1 } @@ -143,7 +166,7 @@ echo "" print_info "Step 5/6: Updating repository database..." "$BUILD_ROOT/bin/update-repo" --arch "$ARCH" --mirror "$MIRROR" || { print_error "Database update failed" - notify_error "Release failed: Database update step failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: Database update step failed" "$RELEASE_CONTEXT" exit 1 } @@ -159,7 +182,7 @@ if [[ "$SKIP_PROD_CHECK" == true ]]; then fi "$BUILD_ROOT/bin/sync-repo" "${SYNC_ARGS[@]}" || { print_error "Sync failed" - notify_error "Release failed: Sync step failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: Sync step failed" "$RELEASE_CONTEXT" exit 1 } @@ -175,7 +198,7 @@ if [[ "$MIRROR" == "stable" ]]; then [[ "$SKIP_PROD_CHECK" == true ]] && REPLICATE_ARGS+=(--skip-prod-check) "$BUILD_ROOT/bin/advance-channel" "${REPLICATE_ARGS[@]}" || { print_error "Fast-ring replication to rc failed" - notify_error "Release failed: rc parity replication failed" "Mirror: $MIRROR | Arch: $ARCH" + notify_error "Release failed: rc parity replication failed" "$RELEASE_CONTEXT" exit 1 } else @@ -185,3 +208,17 @@ fi echo "" print_success "Release workflow completed successfully!" + +duration=$(format_duration $(($(date +%s) - RELEASE_STARTED_AT))) +summary="$RELEASE_CONTEXT
Duration: $duration" +if ((BUILT_COUNT > 0)); then + summary+="

$BUILT_COUNT package(s) published:" + summary+="$(format_package_list_html "$BUILT_FILES")" + summary+="

Live at https://pkgs.omarchy.org/$MIRROR/$ARCH/" + notify_success "Release published: $MIRROR" "$summary" +else + # Nothing to build usually means a queued run raced another that already did + # the work. Worth saying, but not worth dressing up as a publish. + summary+="
No packages needed building β€” the channel was already current." + notify_info "Release complete (no changes): $MIRROR" "$summary" +fi diff --git a/bin/setup b/bin/setup index 78a32a5..a12559e 100755 --- a/bin/setup +++ b/bin/setup @@ -274,6 +274,22 @@ else echo " services source it before signing." fi +# Release reporting is optional, but silence is indistinguishable from a +# broken hook, so say which chat (if any) will receive build reports. +if [[ -f "$CREDENTIALS" ]]; then + if grep -q 'OMARCHY_RELEASE_CHATBOT_URL' "$CREDENTIALS"; then + print_success "Release reports go to the dedicated release chat" + elif grep -q 'BASECAMP_CHATBOT_URL' "$CREDENTIALS"; then + print_warning "Release reports fall back to BASECAMP_CHATBOT_URL (the repository chat)" + echo " Export OMARCHY_RELEASE_CHATBOT_URL in $CREDENTIALS to send them" + echo " to their own chat instead." + else + print_warning "No chatbot URL in $CREDENTIALS β€” build reports are disabled" + echo " Export OMARCHY_RELEASE_CHATBOT_URL to receive start/publish/failure" + echo " reports in Basecamp." + fi +fi + if rclone listremotes 2>/dev/null | grep -q '^pkgs.omarchy.org:'; then print_success "rclone remote 'pkgs.omarchy.org' configured" else diff --git a/helpers/basecamp-notifier.sh b/helpers/basecamp-notifier.sh index 24f7746..98470e2 100644 --- a/helpers/basecamp-notifier.sh +++ b/helpers/basecamp-notifier.sh @@ -1,13 +1,24 @@ # Send notifications to Basecamp Campfire via chatbot # -# Requires BASECAMP_CHATBOT_URL environment variable to be set. -# This is the full chatbot lines URL from Basecamp, e.g.: +# Two destinations, so release traffic does not drown the repository chat: +# +# OMARCHY_RELEASE_CHATBOT_URL release activity (starts, publishes, failures) +# BASECAMP_CHATBOT_URL everything else, and the fallback when the +# release URL is unset +# +# Each is the full chatbot lines URL from Basecamp, e.g. # https://3.basecamp.com//integrations//buckets//chats//lines # -# If BASECAMP_CHATBOT_URL is not set, notifications are silently skipped. +# With neither set, notifications are silently skipped. + +release_chatbot_url() { + echo "${OMARCHY_RELEASE_CHATBOT_URL:-${BASECAMP_CHATBOT_URL:-}}" +} notify_basecamp() { local content="$1" + local BASECAMP_CHATBOT_URL + BASECAMP_CHATBOT_URL=$(release_chatbot_url) if [[ -z "${BASECAMP_CHATBOT_URL:-}" ]]; then return 0 @@ -60,3 +71,89 @@ notify_error() { notify_basecamp "$content" } + +# Human-friendly elapsed time: "8m 12s", "45s", "1h 4m". +format_duration() { + local seconds="${1:-0}" + if ((seconds >= 3600)); then + printf '%dh %dm' $((seconds / 3600)) $(((seconds % 3600) / 60)) + elif ((seconds >= 60)); then + printf '%dm %ds' $((seconds / 60)) $((seconds % 60)) + else + printf '%ds' "$seconds" + fi +} + +notify_start() { + local title="$1" + local details="${2:-}" + + local content="πŸ”¨ ${title}" + [[ -n "$details" ]] && content+="
${details}" + notify_basecamp "$content" +} + +notify_success() { + local title="$1" + local details="${2:-}" + + local content="βœ… ${title}" + [[ -n "$details" ]] && content+="
${details}" + notify_basecamp "$content" +} + +notify_info() { + local title="$1" + local details="${2:-}" + + local content="πŸ“¦ ${title}" + [[ -n "$details" ]] && content+="
${details}" + notify_basecamp "$content" +} + +# "name-1.2.3-1-x86_64.pkg.tar.zst" -> "name 1.2.3-1". Package file names put +# the architecture last and the pkgrel before it, so peel from the right. +package_file_label() { + local file="$1" stem arch rest rel name_version version name + stem="${file%.pkg.tar.*}" + arch="${stem##*-}" + rest="${stem%-*}" + rel="${rest##*-}" + name_version="${rest%-*}" + version="${name_version##*-}" + name="${name_version%-*}" + printf '%s %s-%s' "$name" "$version" "$rel" +} + +# Packages sitting in a build-output directory, newest build first. Excludes +# signatures and the scratch database the builder keeps alongside them. +built_package_files() { + local dir="$1" + [[ -d "$dir" ]] || return 0 + find "$dir" -maxdepth 1 -name '*.pkg.tar.*' ! -name '*.sig' -printf '%f\n' 2>/dev/null | + grep -v '^omarchy-build\.' | sort +} + +# An HTML list of built packages, capped so a 60-package rebuild does not +# produce an unreadable wall of text in chat. +format_package_list_html() { + local files="$1" + local limit="${2:-25}" + local count shown=0 line out="" + + count=$(grep -c '' <<<"$files") + [[ -z "$files" ]] && count=0 + ((count == 0)) && return 0 + + while IFS= read -r line; do + [[ -z "$line" ]] && continue + ((shown >= limit)) && break + out+="
β€’ $(package_file_label "$line" | basecamp_html_escape)" + shown=$((shown + 1)) + done <<<"$files" + + if ((count > shown)); then + out+="
β€’ …and $((count - shown)) more" + fi + echo "$out" +}