Every pull request now builds the package directories it touches on ephemeral DigitalOcean droplets, and every merge to master publishes the resulting artifacts into the channels each package belongs to. The repository host's timers become the fallback rather than the pipeline. Build (.github/workflows/build-pr.yml) One job per package per architecture, always against edge. The artifact is labelled with the package directory's git tree hash. Tooling (bin/, helpers/, build/) is checked out from the base branch; the PR supplies only pkgbuilds/, so a PR can change what is built, never how. Builds run only for trusted authors: collaborators, .github/VOUCHED.td, or a PR carrying the build-approved label. A single required check, result, aggregates the matrix. Publish (.github/workflows/publish.yml, bin/publish-artifact) One job per merge. It collects the PR artifacts for the merged tree, builds anything that has none, then walks each channel/architecture slot once: pull that database, repo-add every package that belongs in it, upload packages, signatures, then the database. A published filename is immutable; identical bytes under an existing name only gain a database entry, different bytes are refused. Fast-ring packages reach edge, rc and stable in the same run from the same file. Matrix (bin/build-matrix) Package x architecture, with the channels the artifact ships to, decided by package_builds_for_mirror so CI and the host agree. arch=any packages build once and land in every architecture database. Builder (build/build.sh, bin/build, build/Dockerfile) With no local published tree, plan against and resolve from the public channel over HTTPS. PACKAGER is set to Omarchy. rclone is in the image. Runners (ci/) A controller droplet polls GitHub with curl and creates one g5 droplet per queued job from cloud-init, deleting them when off or over-age. Builders carry QEMU with credential support for aarch64. Operator SSH keys come from GitHub. tests/controller.sh and tests/publish-artifact.sh cover the decisions against fixtures and real makepkg output. Tests run on pull requests only; branch protection requires result, self-tests and build-isolation with up-to-date branches.
359 lines
13 KiB
Python
Executable File
359 lines
13 KiB
Python
Executable File
#!/bin/bash
|
|
|
|
# Abort if anything fails
|
|
set -e
|
|
|
|
# Source common functions
|
|
BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
|
|
source "$BUILD_ROOT/helpers/message-helpers.sh"
|
|
source "$BUILD_ROOT/helpers/docker-helpers.sh"
|
|
source "$BUILD_ROOT/helpers/paths.sh"
|
|
|
|
print_header "Omarchy Package Builder"
|
|
|
|
DRY_RUN=false
|
|
|
|
# Knobs for builds driven from CI or resumed by hand. Each defaults to the
|
|
# historical behaviour, so an unadorned `bin/build` is unchanged.
|
|
#
|
|
# OMARCHY_KEEP_BUILD_WORKSPACE=1 keep build-output/$MIRROR/$ARCH instead of
|
|
# wiping it, so packages built by an earlier
|
|
# job (or a previous, interrupted run) seed
|
|
# the build database and resolve as
|
|
# dependencies of what builds now.
|
|
# OMARCHY_SKIP_BUILDER_IMAGE=1 use the omarchy-pkg-builder image already
|
|
# present instead of building it; a workflow
|
|
# that builds the image once with an external
|
|
# BuildKit cache can then fan out over many
|
|
# package jobs without each one rebuilding it.
|
|
# OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy/omarchy-settings pair
|
|
# with --nodeps (see build/build.sh); only
|
|
# for a pipeline that verifies the install
|
|
# transaction afterwards.
|
|
KEEP_BUILD_WORKSPACE=${OMARCHY_KEEP_BUILD_WORKSPACE:-0}
|
|
SKIP_BUILDER_IMAGE=${OMARCHY_SKIP_BUILDER_IMAGE:-0}
|
|
DEFER_RUNTIME_DEPS=${OMARCHY_DEFER_RUNTIME_DEPS:-false}
|
|
|
|
# Parse command line arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--arch)
|
|
ARCH="$2"
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--mirror)
|
|
MIRROR="$2"
|
|
if ! validate_mirror "$MIRROR"; then
|
|
print_error "Invalid mirror: $MIRROR (must be one of: $VALID_MIRRORS)"
|
|
exit 1
|
|
fi
|
|
update_arch_paths
|
|
shift 2
|
|
;;
|
|
--package)
|
|
shift
|
|
PACKAGES=""
|
|
while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do
|
|
PACKAGES="$PACKAGES $1"
|
|
shift
|
|
done
|
|
PACKAGES="${PACKAGES# }"
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
-h | --help)
|
|
echo "Usage: $0 [OPTIONS]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --arch <arch> Target architecture (x86_64 or aarch64, default: x86_64)"
|
|
echo " --mirror <mirror> Mirror to use (edge, rc, or stable, default: edge)"
|
|
echo " --package <names> Build only the specified package(s) (space-separated)"
|
|
echo " --dry-run Show what would build without running makepkg"
|
|
echo " -h, --help Show this help message"
|
|
echo ""
|
|
echo "This script builds packages from pkgbuilds/ based on .omarchy/package.json:"
|
|
echo " --mirror edge: builds packages not marked skip_build=true"
|
|
echo " --mirror stable: builds fast-ring packages not marked skip_build=true"
|
|
echo " --package: explicitly builds selected packages, even with skip_build=true"
|
|
echo ""
|
|
echo "Or build specific packages with --package:"
|
|
echo " Package names should match directories in pkgbuilds/"
|
|
echo ""
|
|
echo "Examples:"
|
|
echo " $0 --arch aarch64"
|
|
echo " $0 --mirror stable"
|
|
echo " $0 --package yay"
|
|
echo " $0 --package yay elephant cursor-bin"
|
|
echo ""
|
|
echo "Environment (for CI and resumed builds; defaults keep today's behaviour):"
|
|
echo " OMARCHY_KEEP_BUILD_WORKSPACE=1 keep build-output and reuse packages already there"
|
|
echo " OMARCHY_SKIP_BUILDER_IMAGE=1 use the existing builder image instead of building it"
|
|
echo " OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy pair with --nodeps (transaction verified later)"
|
|
echo " OMARCHY_PUBLISHED_REPO_URL=<url> channel to plan and resolve against when no local tree exists"
|
|
echo " (default https://pkgs.omarchy.org; empty disables the fallback)"
|
|
echo ""
|
|
exit 0
|
|
;;
|
|
*)
|
|
print_error "Unknown option: $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
require_valid_arch "$ARCH"
|
|
|
|
if [[ $DEFER_RUNTIME_DEPS != "false" && $DEFER_RUNTIME_DEPS != "true" ]]; then
|
|
print_error "OMARCHY_DEFER_RUNTIME_DEPS must be true or false"
|
|
exit 1
|
|
fi
|
|
|
|
# Deferring runtime dependencies is only sound for the omarchy pair, and only
|
|
# when both halves are built together: the pair depends on each other and on
|
|
# packages that a sharded pipeline builds in other jobs, and the consumer of
|
|
# this mode installs the assembled set in one verified transaction. Check the
|
|
# request here so a misuse fails before Docker starts.
|
|
if [[ $DEFER_RUNTIME_DEPS == "true" ]]; then
|
|
deferred_runtime=0
|
|
deferred_settings=0
|
|
deferred_count=0
|
|
for package in $PACKAGES; do
|
|
((deferred_count += 1))
|
|
case $package in
|
|
omarchy|omarchy-dev) deferred_runtime=1 ;;
|
|
omarchy-settings|omarchy-settings-dev) deferred_settings=1 ;;
|
|
*)
|
|
print_error "OMARCHY_DEFER_RUNTIME_DEPS only applies to the omarchy/omarchy-settings pair, not $package"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
if (( deferred_runtime != 1 || deferred_settings != 1 || deferred_count != 2 )); then
|
|
print_error "OMARCHY_DEFER_RUNTIME_DEPS requires --package with exactly the omarchy pair"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Show target architecture and mirror after parsing args
|
|
print_info "Target architecture: $ARCH"
|
|
print_info "Mirror: $MIRROR"
|
|
print_info "Build workspace: $BUILD_OUTPUT_DIR"
|
|
print_info "Final output: $REPO_DIR"
|
|
if [[ $DEFER_RUNTIME_DEPS == "true" ]]; then
|
|
print_info "Runtime dependency checks: deferred to the install transaction"
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
print_warning "DRY RUN MODE - build plan only; no container or makepkg will run"
|
|
ARCH="$ARCH" \
|
|
MIRROR="$MIRROR" \
|
|
PACKAGES="$PACKAGES" \
|
|
DRY_RUN=true \
|
|
DEFER_RUNTIME_DEPS="$DEFER_RUNTIME_DEPS" \
|
|
PKGBUILDS_DIR="$PKGBUILDS_DIR" \
|
|
BUILD_OUTPUT_DIR="$BUILD_OUTPUT_DIR" \
|
|
FINAL_OUTPUT_DIR="$REPO_DIR" \
|
|
HELPERS_DIR="$BUILD_ROOT/helpers" \
|
|
SRC_DIR="$SRC_DIR" \
|
|
"$BUILD_ROOT/build/build.sh"
|
|
exit $?
|
|
fi
|
|
|
|
# Create directories if they don't exist
|
|
mkdir -p "$BUILD_OUTPUT_DIR" "$REPO_DIR" "$SRC_DIR"
|
|
|
|
# Check the selected container engine is available
|
|
check_engine
|
|
|
|
# A foreign target architecture runs under QEMU user emulation. Probe by
|
|
# actually running a container for the target platform: that is the only
|
|
# test that covers both "binfmt not registered" and "registered but broken".
|
|
HOST_ARCH=$(uname -m)
|
|
[[ "$HOST_ARCH" == "arm64" ]] && HOST_ARCH=aarch64
|
|
if [[ "$HOST_ARCH" != "$ARCH" ]]; then
|
|
PROBE_IMAGE="alpine:3.21"
|
|
[[ "$CONTAINER_ENGINE" == "podman" ]] && PROBE_IMAGE="docker.io/library/alpine:3.21"
|
|
|
|
# Rootless Podman cannot repair host binfmt state itself. Validate the flags
|
|
# before the basic probe, because an F-only registration can start an ARM
|
|
# container but silently breaks sudo inside it.
|
|
if [[ "$CONTAINER_ENGINE" == "podman" ]]; then
|
|
setup_qemu "$ARCH"
|
|
fi
|
|
|
|
if ! "$CONTAINER_ENGINE" run "${CONTAINER_RUN_ARGS[@]}" --rm "$(get_platform_arg "$ARCH")" "$PROBE_IMAGE" /bin/true >/dev/null 2>&1; then
|
|
if [[ "$CONTAINER_ENGINE" == "podman" ]]; then
|
|
print_error "QEMU $ARCH is registered, but the container probe failed"
|
|
print_info "Refresh the registration with: sudo systemctl restart systemd-binfmt"
|
|
exit 1
|
|
else
|
|
print_info "Setting up QEMU for $ARCH emulation on this $HOST_ARCH host..."
|
|
setup_qemu "$ARCH"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Clean build-output directory to start fresh, unless the caller seeded it
|
|
# with packages from an earlier job or is resuming an interrupted run.
|
|
if [[ $KEEP_BUILD_WORKSPACE == "1" ]]; then
|
|
print_info "Keeping existing build workspace..."
|
|
else
|
|
print_info "Cleaning build workspace..."
|
|
rm -rf "${BUILD_OUTPUT_DIR:?}"/*
|
|
fi
|
|
mkdir -p "$BUILD_OUTPUT_DIR"
|
|
|
|
# Show package info
|
|
if [[ -n "$PACKAGES" ]]; then
|
|
print_info "Building packages: $PACKAGES"
|
|
else
|
|
print_info "Building unscoped packages for $MIRROR mirror"
|
|
fi
|
|
|
|
# Build/update the Docker image, unless the caller prepared the exact image
|
|
# already (a workflow building it once with an external BuildKit cache). A
|
|
# missing image is an error rather than a silent rebuild: the point of the
|
|
# flag is that every job runs the same bytes.
|
|
IMAGE_TAG="omarchy-pkg-builder:latest-$ARCH-$MIRROR"
|
|
if [[ $SKIP_BUILDER_IMAGE == "1" ]]; then
|
|
if ! "$CONTAINER_ENGINE" image inspect "$IMAGE_TAG" >/dev/null 2>&1; then
|
|
print_error "Prepared builder image is unavailable: $IMAGE_TAG"
|
|
exit 1
|
|
fi
|
|
print_info "Using prepared builder image: $IMAGE_TAG"
|
|
else
|
|
build_docker_image "$BUILD_DIR" "$ARCH" "$MIRROR"
|
|
fi
|
|
|
|
print_info "Planning isolated package builds..."
|
|
|
|
# Create output directories if they don't exist
|
|
mkdir -p "$BUILD_OUTPUT_DIR"
|
|
mkdir -p "$REPO_DIR"
|
|
|
|
# Share downloaded archives, never /var/lib/pacman or an installed root.
|
|
# Channel/architecture separation preserves each mirror's dependency set.
|
|
PACKAGE_CACHE_DIR="$BUILD_ROOT/cache/pacman/$MIRROR/$ARCH"
|
|
mkdir -p "$PACKAGE_CACHE_DIR"
|
|
PLAN_DIR=$(mktemp -d "$SRC_DIR/build-plan.XXXXXX")
|
|
trap 'rm -rf "$PLAN_DIR"' EXIT
|
|
# Keep manifest directories host-owned so cleanup also works when Docker's
|
|
# builder uid differs from the caller (as on GitHub runners).
|
|
mkdir -p "$PLAN_DIR/artifacts"
|
|
|
|
# Rootful Docker writes as the image uid, so retain its existing permission
|
|
# workaround. Rootless Podman uses keep-id and must leave ownership/modes alone.
|
|
if [[ "$CONTAINER_ENGINE" == "docker" ]]; then
|
|
make_dir_writable "$BUILD_OUTPUT_DIR"
|
|
make_dir_writable "$PLAN_DIR"
|
|
fi
|
|
|
|
# Build Docker arguments
|
|
DOCKER_ARGS=(
|
|
--rm
|
|
-e ARCH="$ARCH"
|
|
-e MIRROR="$MIRROR"
|
|
-e PACKAGES="$PACKAGES"
|
|
-e OMARCHY_RC_PINS="${OMARCHY_RC_PINS:-}"
|
|
-e OMARCHY_PUBLISHED_REPO_URL="${OMARCHY_PUBLISHED_REPO_URL-https://pkgs.omarchy.org}"
|
|
-e DEFER_RUNTIME_DEPS="$DEFER_RUNTIME_DEPS"
|
|
-e BUILD_PLAN_DIR=/build-plan
|
|
-v "$PLAN_DIR:/build-plan"
|
|
-v "$PACKAGE_CACHE_DIR:/var/cache/pacman/pkg"
|
|
-v "$BUILD_ROOT/build-output:/build-output"
|
|
-v "$REPO_ROOT:/pkgs.omarchy.org:ro"
|
|
-v "$BUILD_DIR:/build:ro"
|
|
-v "$BUILD_ROOT/helpers:/helpers:ro"
|
|
-v "$BUILD_ROOT/pkgbuilds:/pkgbuilds:ro"
|
|
)
|
|
|
|
# Podman-created images can leave WORKDIR owned by a remapped uid. Mount the
|
|
# existing host-user-owned workspace so the builder can write there.
|
|
if [[ "$CONTAINER_ENGINE" == "podman" ]]; then
|
|
DOCKER_ARGS+=(-v "$SRC_DIR:/src")
|
|
fi
|
|
|
|
# Plan once against the published database, then keep that order throughout
|
|
# the run. Each package sees the staged artifacts but starts with a fresh
|
|
# pacman database and root filesystem, even after a failed build.
|
|
PLATFORM_ARG=$(get_platform_arg "$ARCH")
|
|
|
|
"$CONTAINER_ENGINE" run "${CONTAINER_RUN_ARGS[@]}" "$PLATFORM_ARG" "${DOCKER_ARGS[@]}" \
|
|
-e DRY_RUN=true "$IMAGE_TAG" /build/build.sh
|
|
|
|
mapfile -t ORDERED_PACKAGES < "$PLAN_DIR/packages"
|
|
mapfile -t SKIPPED_PACKAGES < "$PLAN_DIR/skipped"
|
|
SUCCESSFUL_PACKAGES=()
|
|
FAILED_PACKAGES=()
|
|
BLOCKED_PACKAGES=()
|
|
declare -A BUILD_STATUS=()
|
|
|
|
for package in "${ORDERED_PACKAGES[@]}"; do
|
|
blocked_by=""
|
|
while read -r consumer dependency; do
|
|
if [[ "$consumer" == "$package" && "${BUILD_STATUS[$dependency]:-}" != success ]]; then
|
|
blocked_by="$dependency"
|
|
break
|
|
fi
|
|
done < "$PLAN_DIR/dependencies"
|
|
|
|
if [[ -n "$blocked_by" ]]; then
|
|
print_warning "$package blocked by unsuccessful dependency: $blocked_by"
|
|
BUILD_STATUS[$package]=blocked
|
|
BLOCKED_PACKAGES+=("$package")
|
|
continue
|
|
fi
|
|
|
|
print_info "Building $package in a fresh container..."
|
|
if "$CONTAINER_ENGINE" run "${CONTAINER_RUN_ARGS[@]}" "$PLATFORM_ARG" "${DOCKER_ARGS[@]}" \
|
|
-e BUILD_PACKAGE="$package" "$IMAGE_TAG" /build/build.sh; then
|
|
BUILD_STATUS[$package]=success
|
|
SUCCESSFUL_PACKAGES+=("$package")
|
|
else
|
|
BUILD_STATUS[$package]=failed
|
|
FAILED_PACKAGES+=("$package")
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
print_header "Build Summary"
|
|
echo " Total packages: ${#ORDERED_PACKAGES[@]}"
|
|
echo " Built: ${#SUCCESSFUL_PACKAGES[@]}"
|
|
echo " Skipped: ${#SKIPPED_PACKAGES[@]} (up-to-date or excluded)"
|
|
echo " Failed: ${#FAILED_PACKAGES[@]}"
|
|
echo " Blocked: ${#BLOCKED_PACKAGES[@]}"
|
|
|
|
# The release caller supplies a fresh directory. A completed result
|
|
# distinguishes package failures from an interrupted/failed orchestrator;
|
|
# only artifacts belonging to fully successful builds may be published.
|
|
if [[ -n "${OMARCHY_BUILD_RESULT_DIR:-}" ]]; then
|
|
mkdir -p "$OMARCHY_BUILD_RESULT_DIR"
|
|
: > "$OMARCHY_BUILD_RESULT_DIR/artifacts"
|
|
for package in "${SUCCESSFUL_PACKAGES[@]}"; do
|
|
cat "$PLAN_DIR/artifacts/$package" >> "$OMARCHY_BUILD_RESULT_DIR/artifacts"
|
|
done
|
|
printf '%s\n' "${FAILED_PACKAGES[@]}" | sed '/^$/d' > "$OMARCHY_BUILD_RESULT_DIR/failed"
|
|
printf '%s\n' "${BLOCKED_PACKAGES[@]}" | sed '/^$/d' > "$OMARCHY_BUILD_RESULT_DIR/blocked"
|
|
touch "$OMARCHY_BUILD_RESULT_DIR/complete"
|
|
fi
|
|
|
|
if (( ${#FAILED_PACKAGES[@]} + ${#BLOCKED_PACKAGES[@]} )); then
|
|
if (( ${#FAILED_PACKAGES[@]} )); then
|
|
echo "Failed packages:"
|
|
printf ' - %s\n' "${FAILED_PACKAGES[@]}"
|
|
fi
|
|
if (( ${#BLOCKED_PACKAGES[@]} )); then
|
|
echo "Packages blocked by failed dependencies:"
|
|
printf ' - %s\n' "${BLOCKED_PACKAGES[@]}"
|
|
fi
|
|
print_warning "Some packages failed (see details above)"
|
|
# Reserved for a completed run with unsuccessful packages. Other failures
|
|
# must not let release publish arbitrary files left in the workspace.
|
|
exit 2
|
|
fi
|
|
|
|
print_success "Build completed successfully!"
|