omarchy-iso/.github/workflows/nightly-build.yml

name: Nightly ISO Build

# Every night, for two channels: installs an ISO unattended and boots what it
# installed (the install test), turns that installed disk into a VM image, and
# publishes it with a latest.json saying what is in it.
#
#   edge    the ISO is built here, from quattro and the edge packages
#   stable  the ISO is the last release: omacom/omarchy's latest release names
#           the version, iso.omarchy.org has the ISO, its signature is checked.
#           Done once for each release, skipped on the nights after.
#
#   https://nightly.omarchy.org/<channel>/latest.json
#   https://nightly.omarchy.org/edge/<version>/omarchy-edge-<version>.iso
#   https://nightly.omarchy.org/edge/<version>/vm/        base.qcow2 and what boots it
#   https://nightly.omarchy.org/stable/<version>/<run>/vm/
#
# Nothing is published unless the ISO installed and the image passed its
# checks. A version's files are written once and never replaced: the CDN keeps
# them for a year. Only latest.json changes, and it is written last.
#
# No pull_request trigger: this workflow holds the key to the bucket, so it
# never runs for a fork. To try a change, run it by hand on a branch; that
# builds and checks everything and publishes nothing.

on:
  schedule:
    - cron: "1 0 * * *" # Daily at 00:01 UTC
  workflow_dispatch:
    inputs:
      publish:
        description: "Publish to nightly.omarchy.org (only from quattro)"
        type: boolean
        default: false

permissions:
  contents: read

concurrency:
  group: nightly-${{ github.ref }}
  cancel-in-progress: false

env:
  SITE: https://nightly.omarchy.org

jobs:
  # Which channels this run makes, under which version, and where each goes.
  # An edge version is the date and the run number. A stable version is the
  # release's own, and a release already published is left out of a scheduled
  # run. Two builds of one release are not the same bytes, so a stable image
  # also has the run number in its path: a published path is never reused, and
  # a publish that stopped halfway is simply made again.
  plan:
    name: Plan
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    outputs:
      channels: ${{ steps.plan.outputs.channels }}
    steps:
      - id: plan
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          # A run that is started again makes its images again, and they are
          # not the same bytes: it gets its own number too.
          build=${{ github.run_number }}
          [[ ${{ github.run_attempt }} == 1 ]] || build+=.${{ github.run_attempt }}
          edge=$(date -u +%Y.%m.%d).$build
          stable=$(gh api repos/omacom/omarchy/releases/latest --jq .tag_name)
          stable=${stable#v}
          [[ $stable =~ ^[0-9]+(\.[0-9]+)+$ ]] || { echo "::error::omacom/omarchy's latest release is '$stable', not a version"; exit 1; }
          published=$(curl -fsL "$SITE/stable/latest.json?run=$GITHUB_RUN_ID" | jq -r .version || true)
          channels=$(jq -cn --arg edge "$edge" '[{channel: "edge", version: $edge, release: "edge/\($edge)"}]')
          if [[ $GITHUB_EVENT_NAME == schedule && $published == "$stable" ]]; then
            echo "stable $stable is already published"
          else
            channels=$(jq -c --arg stable "$stable" --arg run "$build" \
              '. + [{channel: "stable", version: $stable, release: "stable/\($stable)/\($run)"}]' <<<"$channels")
          fi
          echo "channels=$channels" | tee -a "$GITHUB_OUTPUT"

  build:
    name: ${{ matrix.channel }} ${{ matrix.version }}
    needs: plan
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.plan.outputs.channels) }}
    runs-on: namespace-profile-linux-amd64-16x32
    timeout-minutes: 90
    env:
      CHANNEL: ${{ matrix.channel }}
      VERSION: ${{ matrix.version }}
      RELEASE: ${{ matrix.release }}
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5

      - name: Runner
        run: |
          uname -srm
          echo "$(nproc) CPUs, $(free -h | awk 'NR==2 {print $2}') memory"
          grep -m1 'model name' /proc/cpuinfo || true
          echo "LSMs: $(cat /sys/kernel/security/lsm 2>/dev/null || echo unreadable)"
          ls -l /dev/kvm

      - name: Check disk space
        run: |
          df -h /
          avail_gb=$(df -BG --output=avail / | tail -1 | tr -dc 0-9)
          [ "$avail_gb" -ge 50 ] || { echo "::error::only ${avail_gb} GB free, the build, the install test and the image need ~50 GB"; exit 1; }

      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
        if: matrix.channel == 'edge'

      # What a build downloads and would download again tomorrow: the packages
      # the ISO's offline mirror is made of (the build prunes the ones it no
      # longer needs) and the test container's own. Skipped, not failed, on a
      # runner without a cache volume.
      - name: Package cache (Namespace)
        continue-on-error: true
        uses: namespacelabs/nscloud-cache-action@c5f8dab7560444c4bf8dbc64f1b203431873c547 # v1.6.1
        with:
          path: |
            ~/.cache/omarchy
            ~/.cache/omarchy-ci

      - name: Build the edge ISO
        if: matrix.channel == 'edge'
        run: NO_BOOT_OFFER=1 ./bin/omarchy-iso-make --edge

      # Good only if the signature is the release key's, the one in this repo.
      - name: Download the released ISO and check its signature
        if: matrix.channel == 'stable'
        run: |
          set -euo pipefail
          mkdir -p release
          iso=omarchy-$VERSION.iso
          curl -fSL --retry 3 -o "release/$iso" "https://iso.omarchy.org/$iso"
          curl -fsSL --retry 3 -o "release/$iso.sig" "https://iso.omarchy.org/$iso.sig"
          home=$(mktemp -d)
          gpg --homedir "$home" --quiet --batch --import builder/omarchy.gpg
          key=$(gpg --homedir "$home" --batch --with-colons --list-keys | awk -F: '$1 == "fpr" { print $10; exit }')
          gpg --homedir "$home" --batch --status-fd 1 --verify "release/$iso.sig" "release/$iso" 2>/dev/null |
            grep -q "^\[GNUPG:\] VALIDSIG $key " || { echo "::error::$iso is not signed by $key"; exit 1; }
          echo "$iso is signed by $key"
          rm -rf "$home" "release/$iso.sig"

      # The same install test a pull request gets (install-test.yml), then the
      # image: the installed disk that test leaves behind, sealed and checked.
      - name: Install unattended, boot the installed system, seal it
        run: |
          sudo chmod 0666 /dev/kvm
          mkdir -p ~/.cache/omarchy-ci/pacman
          docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
            -v "$HOME/.cache/omarchy-ci/pacman:/var/cache/pacman/pkg" \
            -e OMARCHY_INTEGRATION_DISK_CACHE=none \
            archlinux:latest bash -euo pipefail -c '
              # pacman 7 fails every sync on a kernel without Landlock, which
              # these runners have; builder/build-iso.sh does the same.
              if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
                printf "#!/bin/sh\nexec /usr/bin/pacman --disable-sandbox \"\$@\"\n" >/usr/local/bin/pacman
                chmod 0755 /usr/local/bin/pacman
              fi
              pacman -Syu --noconfirm --needed qemu-system-x86 qemu-hw-display-virtio-gpu qemu-hw-display-virtio-vga qemu-img edk2-ovmf socat \
                imagemagick tesseract tesseract-data-eng mtools dosfstools jq openssh openssl python curl binutils >/dev/null
              # test/integration installs its tools through Omarchy'"'"'s helper; they are installed above.
              printf "#!/bin/sh\nexit 0\n" >/usr/local/bin/omarchy-pkg-add
              chmod 0755 /usr/local/bin/omarchy-pkg-add
              qemu-system-x86_64 --version | head -1
              ./test/integration release/*.iso boot --no-preview --memory 8192 --timeout 1800

              base=$(ls -d test-runs/*-integration)
              ./test/vm-image/vm-seal "$base" sealed
              ./test/vm-image/vm-verify sealed | tee vm-verify.txt
              ./test/vm-image/vm-boot sealed "pacman -Qq omarchy omarchy-dev 2>/dev/null | xargs pacman -Q; pacman -Qo /usr/lib/modules/\$(uname -r)/pkgbase; uname -r" >vm-packages.txt
              cat vm-packages.txt
            '

      # The container ran as root; what it wrote belongs to root until here.
      - name: Hand the results to the runner's user
        if: always()
        run: |
          mkdir -p test-runs sealed
          sudo chown -R "$(id -u):$(id -g)" test-runs sealed release vm-*.txt 2>/dev/null || true

      - name: Lay out the release and describe it
        run: |
          set -euo pipefail
          dir=out/$RELEASE
          mkdir -p "$dir/vm"
          # The edge ISO is published here. The released one already has its
          # place, so latest.json points there and it is not uploaded again.
          if [[ $CHANNEL == edge ]]; then
            iso=omarchy-$CHANNEL-$VERSION.iso iso_url=$SITE/$RELEASE/$iso
            mv release/*.iso "$dir/$iso"
            (cd "$dir" && sha256sum "$iso" >"$iso.sha256")
            iso_sha256=$(cut -d" " -f1 "$dir/$iso.sha256") iso_size=$(stat -c %s "$dir/$iso")
          else
            iso=omarchy-$VERSION.iso iso_url=https://iso.omarchy.org/$iso
            iso_sha256=$(sha256sum "release/$iso" | cut -d" " -f1) iso_size=$(stat -c %s "release/$iso")
          fi
          mv sealed/* "$dir/vm/"
          cp test/vm-image/vm-boot "$dir/vm/vm-boot"
          (cd "$dir/vm" && sha256sum base.qcow2 OVMF_VARS.4m.fd vmlinuz initrd cmdline vm-boot >SHA256SUMS)

          omarchy=$(awk '$1 == "omarchy" || $1 == "omarchy-dev" { print $2 }' vm-packages.txt)
          jq -n \
            --arg version "$VERSION" --arg channel "$CHANNEL" --arg site "$SITE/$RELEASE" --arg iso_url "$iso_url" \
            --arg built_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --arg sha256 "$iso_sha256" --argjson size "$iso_size" \
            --arg iso_commit "$GITHUB_SHA" \
            --arg omarchy_version "$omarchy" --arg omarchy_commit "$(sed -nE "s/.*\.g([0-9a-f]+)-.*/\1/p" <<<"$omarchy")" \
            --arg kernel "$(tail -n 1 vm-packages.txt)" \
            --arg disk_sha256 "$(awk '$2 == "base.qcow2" { print $1 }' "$dir/vm/SHA256SUMS")" \
            --argjson disk_size "$(stat -c %s "$dir/vm/base.qcow2")" \
            '{
              version: $version, channel: $channel, built_at: $built_at,
              url: $iso_url, sha256: $sha256, size: $size,
              iso_commit: $iso_commit, omarchy_version: $omarchy_version, omarchy_commit: $omarchy_commit,
              kernel: $kernel,
              vm: {
                url: "\($site)/vm/", disk: "\($site)/vm/base.qcow2", disk_sha256: $disk_sha256, disk_size: $disk_size,
                files: ["base.qcow2", "OVMF_VARS.4m.fd", "vmlinuz", "initrd", "cmdline", "vm-boot", "SHA256SUMS"],
                user: "omarchy", password: "omarchy",
                ssh: "keys only; vm-boot makes one for each boot"
              }
            }' | tee "out/$CHANNEL/latest.json"

          {
            echo "## Nightly $CHANNEL $VERSION"
            echo
            echo "| File | Size |"
            echo "| --- | ---: |"
            (cd "$dir" && find . -type f -printf '| `%P` | %s |\n' | sort) | awk -F'|' '{ printf "|%s| %.0f MiB |\n", $2, $3 / 1048576 }'
            echo
            echo '```'
            cat vm-verify.txt
            echo '```'
          } >>"$GITHUB_STEP_SUMMARY"

      - name: Keep the release for the publish job
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: nightly-${{ matrix.channel }}-${{ github.run_id }}
          path: out/
          compression-level: 0
          retention-days: 2
          if-no-files-found: error

      - name: Keep the install test's logs
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: nightly-install-test-${{ matrix.channel }}-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            test-runs/*/omarchy-install-timing.json
            test-runs/*/omarchy-install.log
            test-runs/*/first-boot-systemd-analyze.txt
            test-runs/*/runs/**/*.png
            test-runs/*/runs/**/*.txt
            test-runs/*/runs/**/*.json
            test-runs/*/runs/**/*.log
            vm-verify.txt
            vm-packages.txt
          if-no-files-found: ignore
          retention-days: 14

  # The only job that holds the bucket's key, and it builds nothing: it copies
  # what the job above made, checks what the CDN then serves, and only then
  # points latest.json at it.
  # One channel failing does not hold back the other: each is published if
  # its own build left a release behind.
  publish:
    name: Publish ${{ matrix.channel }}
    needs: [plan, build]
    if: ${{ !cancelled() && github.ref == 'refs/heads/quattro' && (github.event_name == 'schedule' || inputs.publish) }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.plan.outputs.channels) }}
    runs-on: ubuntu-24.04
    timeout-minutes: 45
    environment: nightly
    env:
      CHANNEL: ${{ matrix.channel }}
      VERSION: ${{ matrix.version }}
      RELEASE: ${{ matrix.release }}
      RCLONE_CONFIG_R2_TYPE: s3
      RCLONE_CONFIG_R2_PROVIDER: Cloudflare
      RCLONE_CONFIG_R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
      RCLONE_CONFIG_R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
      RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
      BUCKET: omarchy-nightly
    steps:
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: nightly-${{ matrix.channel }}-${{ github.run_id }}
          path: out

      - name: rclone
        run: |
          [ -n "$RCLONE_CONFIG_R2_ACCESS_KEY_ID" ] || { echo "::error::the nightly environment has no bucket key"; exit 1; }
          sudo apt-get update -qq && sudo apt-get install -y -qq rclone >/dev/null
          rclone version | head -1

      # --immutable: a name that exists with other bytes is an error, not an
      # overwrite.
      - name: Upload this version
        run: |
          rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
            --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line

      # Fetched the way a user fetches them. It proves what is served, and it
      # is the first request the CDN sees for each file.
      - name: Check what the CDN serves
        run: |
          set -euo pipefail
          dir=out/$RELEASE
          check() { # url, expected sha256
            got=$(curl -fsSL --retry 3 "$1" | sha256sum | cut -d" " -f1)
            [ "$got" = "$2" ] || { echo "::error::$1 serves $got, expected $2"; exit 1; }
            echo "ok $1"
          }
          for iso in "$dir"/*.iso; do
            [[ -f $iso ]] || continue
            check "$SITE/$RELEASE/${iso##*/}" "$(cut -d" " -f1 "$iso.sha256")"
          done
          while read -r sum name; do
            check "$SITE/$RELEASE/vm/$name" "$sum"
          done <"$dir/vm/SHA256SUMS"

      - name: Point latest.json at it
        run: |
          rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
          served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version)
          [ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; }
          echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY"