diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml new file mode 100644 index 0000000..04c62f5 --- /dev/null +++ b/.github/workflows/upgrade-test.yml @@ -0,0 +1,227 @@ +name: Upgrade test + +# What an existing user gets: installs a release from its ISO, moves it to +# another channel the way a user would (omarchy-channel-set), reboots it, and +# checks what came back. test/upgrade-test does the work and says what it +# checks. +# +# Started by hand: from the latest release to edge unless told otherwise. It +# also runs for a pull request that changes the test itself. +# +# When there is a pull request for the branch it ran on, the result goes there +# as one comment, updated in place on later runs, with a picture of the desktop +# before and after. The pictures are committed to the upgrade-test-screenshots +# branch, which is what lets a comment show them. + +on: + workflow_dispatch: + inputs: + to: + description: "Channel to move to" + type: choice + options: [edge, rc] + default: edge + from: + description: "Release to start from, like 4.0.4 (default: the latest)" + type: string + default: "" + pull_request: + paths: + - test/upgrade-test + - .github/workflows/upgrade-test.yml + +permissions: + contents: read + +concurrency: + group: upgrade-test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + TO: ${{ inputs.to || 'edge' }} + +jobs: + upgrade: + name: Install, upgrade, reboot + # Our own branches run on Namespace; a pull request from a fork stays on + # GitHub's runner. + runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-26.04' || 'namespace-profile-linux-amd64-16x32' }} + timeout-minutes: 60 + outputs: + from: ${{ steps.release.outputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - id: release + env: + GH_TOKEN: ${{ github.token }} + FROM: ${{ inputs.from }} + run: | + set -euo pipefail + version=${FROM:-$(gh api repos/omacom/omarchy/releases/latest --jq .tag_name)} + version=${version#v} + [[ $version =~ ^[0-9]+(\.[0-9]+)+$ ]] || { echo "::error::'$version' is not a release version"; exit 1; } + echo "version=$version" | tee -a "$GITHUB_OUTPUT" + + # Good only if the signature is the release key's, the one in this repo. + - name: Download the released ISO and check its signature + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + mkdir -p release + iso=omarchy-$VERSION.iso + curl -fSL --retry 3 -o "release/$iso" "https://iso.omarchy.org/$iso" + curl -fsSL --retry 3 -o "release/$iso.sig" "https://iso.omarchy.org/$iso.sig" + home=$(mktemp -d) + gpg --homedir "$home" --quiet --batch --import builder/omarchy.gpg + key=$(gpg --homedir "$home" --batch --with-colons --list-keys | awk -F: '$1 == "fpr" { print $10; exit }') + gpg --homedir "$home" --batch --status-fd 1 --verify "release/$iso.sig" "release/$iso" 2>/dev/null | + grep -q "^\[GNUPG:\] VALIDSIG $key " || { echo "::error::$iso is not signed by $key"; exit 1; } + echo "$iso is signed by $key" + rm -rf "$home" "release/$iso.sig" + + # The install test's own install, which leaves the installed disk behind; + # the upgrade then starts from that disk. A failed upgrade does not stop + # here: the evidence is kept first, and the last step fails the job. + - name: Install the release, then upgrade it + run: | + sudo chmod 0666 /dev/kvm + docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \ + -e OMARCHY_INTEGRATION_DISK_CACHE=none -e TO \ + ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c ' + if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then + printf "#!/bin/sh\nexec /usr/bin/pacman --disable-sandbox \"\$@\"\n" >/usr/local/bin/pacman + chmod 0755 /usr/local/bin/pacman + fi + sed -i "1i Server = https://mirror.omarchy.org/\$repo/os/\$arch" /etc/pacman.d/mirrorlist + for attempt in 1 2 3; do + pacman -Syu --noconfirm --needed qemu-system-x86 qemu-hw-display-virtio-gpu qemu-hw-display-virtio-vga qemu-img edk2-ovmf socat \ + imagemagick tesseract tesseract-data-eng mtools dosfstools jq openssh openssl python curl >/dev/null && break + [ "$attempt" -lt 3 ] || { echo "::error::the test container could not install its tools"; exit 1; } + echo "pacman failed, attempt $attempt of 3; trying again in 15 s"; sleep 15 + done + printf "#!/bin/sh\nexit 0\n" >/usr/local/bin/omarchy-pkg-add + chmod 0755 /usr/local/bin/omarchy-pkg-add + ./test/integration release/*.iso boot --no-preview --memory 8192 --timeout 1800 + + status=0 + VM_STATE_DIR=/w ./test/upgrade-test "$(ls -d test-runs/*-integration)" "$TO" upgrade-evidence || status=$? + echo "$status" >upgrade-evidence/status + ' + + - name: Hand the results to the runner's user + if: always() + run: sudo chown -R "$(id -u):$(id -g)" upgrade-evidence test-runs 2>/dev/null || true + + - name: Keep the evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} + path: upgrade-evidence/ + if-no-files-found: ignore + retention-days: 14 + + - name: The upgrade's result + run: | + cat upgrade-evidence/report.md >>"$GITHUB_STEP_SUMMARY" + [ "$(cat upgrade-evidence/status)" = 0 ] || { echo "::error::the upgrade test failed, see the summary"; exit 1; } + + # The only job that can write: it commits the two pictures and writes the + # comment. It runs nothing from the test; it handles two PNG files and a + # Markdown report. + report: + name: Report + needs: upgrade + if: ${{ !cancelled() && !github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + env: + GH_TOKEN: ${{ github.token }} + FROM: ${{ needs.upgrade.outputs.from }} + PR: ${{ github.event.pull_request.number }} + BRANCH: ${{ github.head_ref || github.ref_name }} + SHOTS_BRANCH: upgrade-test-screenshots + steps: + # No evidence at all still gets a comment saying so. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + continue-on-error: true + with: + name: upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + + # A comment can only show a picture that has a public address. A commit + # on a branch of this repository gives it one, and the address of a + # commit never changes. + - name: Commit the pictures + id: pictures + run: | + set -euo pipefail + [ -s evidence/before.png ] || [ -s evidence/after.png ] || { echo "no pictures to commit"; exit 0; } + url=https://x-access-token:$GH_TOKEN@github.com/$GITHUB_REPOSITORY.git + git init -q shots && cd shots + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git fetch -q --depth 1 "$url" "$SHOTS_BRANCH" 2>/dev/null; then + git checkout -q -b "$SHOTS_BRANCH" FETCH_HEAD + else + git checkout -q --orphan "$SHOTS_BRANCH" + fi + dir=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT + mkdir -p "$dir" + cp ../evidence/*.png "$dir/" + git add "$dir" + git commit -q -m "Upgrade test $GITHUB_RUN_ID, attempt $GITHUB_RUN_ATTEMPT: $FROM to $TO" + git push -q "$url" "$SHOTS_BRANCH" + echo "base=https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$(git rev-parse HEAD)/$dir" >>"$GITHUB_OUTPUT" + + - name: Write the comment + env: + PICTURES: ${{ steps.pictures.outputs.base }} + RESULT: ${{ needs.upgrade.result }} + run: | + set -euo pipefail + marker="" + { + echo "$marker" + echo "## Upgrade test: $FROM → $TO" + echo + if [ -s evidence/report.md ]; then + cat evidence/report.md + else + echo "**The test did not get as far as a report** (job result: $RESULT)." + fi + if [ -n "$PICTURES" ]; then + echo + echo "| Before, $FROM | After, on $TO |" + echo "| --- | --- |" + echo "| ![before]($PICTURES/before.png) | ![after]($PICTURES/after.png) |" + if [ -s evidence/stuck.png ]; then + echo + echo "The reboot did not come back by itself. This is what the screen showed:" + echo + echo "![stuck]($PICTURES/stuck.png)" + fi + fi + echo + echo "[Run $GITHUB_RUN_ID]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID), attempt $GITHUB_RUN_ATTEMPT, at \`${GITHUB_SHA:0:7}\`." + } >comment.md + cat comment.md >>"$GITHUB_STEP_SUMMARY" + + # A run started by hand has no pull request of its own: use the open + # one for its branch, if there is one. + pr=${PR:-$(gh pr list -R "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq '.[0].number // empty')} + [ -n "$pr" ] || { echo "No pull request for $BRANCH: the result is on this run's page."; exit 0; } + existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$pr/comments" --paginate \ + --jq "[.[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))][0].id // empty") + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@comment.md >/dev/null + echo "Updated the comment on #$pr" + else + gh api "repos/$GITHUB_REPOSITORY/issues/$pr/comments" -F body=@comment.md >/dev/null + echo "Commented on #$pr" + fi diff --git a/README.md b/README.md index cc5fe51..e82eedd 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,25 @@ The first scenario is `factory-reset`: it proves `omarchy-system-factory-reset` Artifacts — screenshots, the fixtured/staged/final `limine.conf`, the reset typescript, and the factory-reset log — land under `test-runs/-integration/runs/-/`, and `--no-preview` skips the `imv` review just like the acceptance harness. +## Upgrade testing + +`test/upgrade-test` checks what an existing user gets. It takes an installed system, moves it to another channel with `omarchy-channel-set`, reboots it, and checks what came back: the update's exit status, skipped migrations, that the reboot comes back by itself into an installed kernel, failed units, the desktop, the user's wallpaper and the boot menu. + +```bash +./test/integration release/omarchy.iso boot # installs once, leaves the installed disk +./test/upgrade-test test-runs/omarchy-integration edge upgrade-evidence # edge or rc +``` + +The evidence folder gets `report.md`, the update's output, and a picture of the desktop before and after. + +The "Upgrade test" workflow does both steps from a release's ISO. Start it by hand, from the latest release to edge by default: + +```bash +gh workflow run upgrade-test.yml -f to=edge +``` + +When the branch it ran on has a pull request, the result is posted there as one comment and updated in place on later runs. + ## Signing the ISO Run `./bin/omarchy-iso-sign [release/omarchy.iso]`. The signing key is retrieved from the shared Omarchy vault with the 1Password CLI. diff --git a/test/upgrade-test b/test/upgrade-test new file mode 100755 index 0000000..1df9e49 --- /dev/null +++ b/test/upgrade-test @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Takes an installed Omarchy, as a user's machine holds it, moves it to another +# channel the way a user would, reboots it, and checks what came back. +# +# test/upgrade-test +# +# installed_dir is what test/integration leaves once its install has passed, +# test-runs/-integration: base.qcow2, OVMF_VARS.4m.fd and id_ed25519. It is +# not written to; the machine runs on an overlay. channel is edge or rc. +# +# The machine boots through the firmware and the boot loader, so the reboot +# after the update starts whatever kernel the update installed. Commands run +# in a login shell, as a user's terminal does: Omarchy's commands need its +# environment. +# +# evidence_dir gets before.png and after.png (the desktop, unlocked), the +# update's output, the facts before and after, and report.md; stuck.png too if +# the reboot did not come back by itself. The exit status is 0 only if every +# check passed. +# +# Needs qemu-system-x86_64, qemu-img, ssh, python3 and a usable /dev/kvm. +# Env: SMP (4), MEM (8G), SSH_PORT (2422), GUEST_PASSWORD (omarchy), +# UPDATE_TIMEOUT (seconds, 1800), EXTRA_MIGRATIONS (migration files to put +# in place before the update, to try a fix that is not released yet). +set -uo pipefail +[[ $# -eq 3 ]] || { sed -n '3,24p' "$0" >&2; exit 2; } +in=$(realpath "$1") channel=$2 out=$(realpath -m "$3") +pw=${GUEST_PASSWORD:-omarchy} port=${SSH_PORT:-2422} +[[ $channel == edge || $channel == rc ]] || { echo "upgrade-test: channel is edge or rc" >&2; exit 2; } +for f in base.qcow2 OVMF_VARS.4m.fd id_ed25519; do + [[ -s $in/$f ]] || { echo "upgrade-test: $in/$f is missing" >&2; exit 1; } +done +[[ -r /dev/kvm && -w /dev/kvm ]] || { echo "upgrade-test: /dev/kvm is not usable" >&2; exit 1; } +for fw in /usr/share/edk2/x64/OVMF_CODE.4m.fd /usr/share/OVMF/OVMF_CODE_4M.fd; do + [[ -f $fw ]] && { fw_code=$fw; break; } +done +: "${fw_code:?no UEFI firmware found}" + +mkdir -p "$out" +state=$(mktemp -d "${VM_STATE_DIR:-/var/tmp}/upgrade-test.XXXXXX") +cleanup() { + local status=$? pid + if [[ -f $state/qemu.pid ]]; then + pid=$(cat "$state/qemu.pid") + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 50); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done + fi + rm -rf "$state" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +log() { printf '==> %s\n' "$*"; } +install -m600 "$in/id_ed25519" "$state/key" +ssh_opts=(-i "$state/key" -p "$port" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=3) +guest() { ssh "${ssh_opts[@]}" omarchy@127.0.0.1 "bash -lc $(printf %q "$1")" /dev/null; do + kill -0 "$(cat "$state/qemu.pid")" 2>/dev/null || { echo "upgrade-test: qemu exited" >&2; return 1; } + (( SECONDS - start < $1 )) || return 1 + sleep 1 + done +} +qmp() { # shot | key ... + python3 - "$state/qmp.sock" "$@" <<'PY' +import json, socket, sys +s = socket.socket(socket.AF_UNIX); s.settimeout(20); s.connect(sys.argv[1]); f = s.makefile("rw"); f.readline() +def run(command): + f.write(json.dumps(command) + "\n"); f.flush() + while "return" not in (reply := json.loads(f.readline())): + if "error" in reply: sys.exit(reply["error"]["desc"]) +run({"execute": "qmp_capabilities"}) +if sys.argv[2] == "shot": + run({"execute": "screendump", "arguments": {"filename": sys.argv[3], "format": "png"}}) +else: + for key in sys.argv[3:]: + run({"execute": "send-key", "arguments": {"keys": [{"type": "qcode", "data": key}]}}) +PY +} +# The session starts locked. Type the password, as its owner would, and wait +# for the desktop to draw. +desktop_shot() { # file + local key keys=() + for (( key = 0; key < ${#pw}; key++ )); do keys+=("${pw:key:1}"); done + qmp key "${keys[@]}" ret || true + sleep 8 + qmp shot "$1" || echo "upgrade-test: no screenshot" >&2 +} +# One fact per line, name=value, the same names before and after. +facts() { # file + guest ' + link=~/.local/state/omarchy/current/background; target=$(readlink -f "$link" 2>/dev/null) + echo "omarchy=$(pacman -Qq omarchy omarchy-dev 2>/dev/null | sort -u | xargs -r pacman -Q 2>/dev/null | paste -sd, -)" + echo "channel=$(omarchy-channel-current 2>/dev/null)" + echo "kernel=$(uname -r)" + echo "kernel_is_installed=$([[ -d /usr/lib/modules/$(uname -r)/kernel ]] && echo yes || echo no)" + echo "systemd=$(systemctl --version | sed -n "1s/^systemd \([0-9.]*\).*/\1/p")" + echo "hyprland=$(pacman -Q hyprland 2>/dev/null | cut -d" " -f2)" + echo "system_state=$(systemctl is-system-running 2>/dev/null)" + echo "failed_units=$(systemctl --failed --no-legend --plain | awk "{print \$1}" | paste -sd, -)" + echo "failed_user_units=$(systemctl --user --failed --no-legend --plain | awk "{print \$1}" | paste -sd, -)" + echo "hyprland_running=$(pgrep -c -x Hyprland)" + echo "migrations_run=$(find ~/.local/state/omarchy/migrations -maxdepth 1 -name "*.sh" | wc -l)" + echo "migrations_skipped=$(find ~/.local/state/omarchy/migrations/skipped -name "*.sh" 2>/dev/null | wc -l)" + echo "wallpaper=${target##*/}" + echo "wallpaper_exists=$([[ -n $target && -e $target ]] && echo yes || echo no)" + echo "boot_entry_groups=$(sudo grep -c "^/+" /boot/limine.conf 2>/dev/null)" + echo "boot_id=$(cut -c1-8 /proc/sys/kernel/random/boot_id)" + ' 2>/dev/null | grep -E '^[a-z_]+=' >"$1" +} +fact() { sed -n "s/^$2=//p" "$1" | head -n 1; } + +qemu-img create -qf qcow2 -b "$in/base.qcow2" -F qcow2 "$state/run.qcow2" +cp "$in/OVMF_VARS.4m.fd" "$state/vars.fd" +: >"$out/serial.log" +qemu-system-x86_64 -cpu host -enable-kvm -machine q35,accel=kvm -smp "${SMP:-4}" -m "${MEM:-8G}" \ + -drive "if=pflash,format=raw,readonly=on,file=$fw_code" -drive "if=pflash,format=raw,file=$state/vars.fd" \ + -drive "file=$state/run.qcow2,format=qcow2,if=none,id=d0" -device virtio-blk-pci,drive=d0 \ + -device virtio-vga -device virtio-rng-pci -display none -usb -device usb-kbd -serial "file:$out/serial.log" \ + -netdev "user,id=n0,hostfwd=tcp:127.0.0.1:$port-:22" -device virtio-net-pci,netdev=n0 \ + -qmp "unix:$state/qmp.sock,server,nowait" -pidfile "$state/qemu.pid" -daemonize + +log "Booting the installed system" +wait_ssh 300 || { echo "upgrade-test: the installed system did not come up" >&2; exit 1; } +# The overlay is thrown away: let sudo through without asking, so a long +# update does not stop at a password prompt nobody is there to answer. +ssh "${ssh_opts[@]}" omarchy@127.0.0.1 "echo $(printf %q "$pw") | sudo -S -p '' sh -c 'echo \"omarchy ALL=(ALL) NOPASSWD: ALL\" >/etc/sudoers.d/99-upgrade-test'" /dev/null 2>&1 || true' +facts "$out/before.txt" +desktop_shot "$out/before.png" +log "Before: $(fact "$out/before.txt" omarchy), kernel $(fact "$out/before.txt" kernel), channel $(fact "$out/before.txt" channel)" + +for migration in ${EXTRA_MIGRATIONS:-}; do + log "Putting $(basename "$migration") in place before the update" + ssh "${ssh_opts[@]}" omarchy@127.0.0.1 "sudo tee /usr/share/omarchy/migrations/$(basename "$migration") >/dev/null" <"$migration" +done + +log "omarchy-channel-set $channel" +update_start=$SECONDS +guest "timeout ${UPDATE_TIMEOUT:-1800} omarchy-channel-set $channel" >"$out/update.log" 2>&1 +update_status=$? +update_seconds=$(( SECONDS - update_start )) +sed -i 's/\x1b\[[0-9;?]*[A-Za-z]//g; s/\r/\n/g' "$out/update.log" +migrations_in_update=$(grep -c '^Running migration' "$out/update.log" || true) +log "The update exited $update_status after $update_seconds s, $migrations_in_update migrations" + +log "Rebooting" +guest 'sudo systemctl reboot' >/dev/null 2>&1 || true +# Wait for the old boot to go away first: its sshd answers for a moment yet. +for _ in $(seq 1 90); do + ssh "${ssh_opts[@]}" -o ConnectTimeout=2 omarchy@127.0.0.1 true /dev/null || break + sleep 1 +done +rebooted=no +if wait_ssh 150; then + rebooted=yes +else + # A boot that sits there until a key is pressed is a failure, and a known + # kind: keep what the screen showed, then press Enter to find out whether + # that was it, and to have a system to collect the rest from. + qmp shot "$out/stuck.png" || true + qmp key ret || true + if wait_ssh 150; then rebooted="only after a key was pressed"; fi +fi +if [[ $rebooted != no ]]; then + guest 'timeout 180 systemctl is-system-running --wait >/dev/null 2>&1 || true' + facts "$out/after.txt" + desktop_shot "$out/after.png" +else + : >"$out/after.txt" + qmp shot "$out/after.png" || true +fi +log "After: $(fact "$out/after.txt" omarchy), kernel $(fact "$out/after.txt" kernel), channel $(fact "$out/after.txt" channel)" + +failures=0 +rows=() +check() { # description, command... + local what=$1; shift + if "$@"; then rows+=("| ✅ | $what |"); else rows+=("| ❌ | $what |"); failures=$((failures + 1)); fi +} +b() { fact "$out/before.txt" "$1"; } +a() { fact "$out/after.txt" "$1"; } +check "The update finished without an error (exit $update_status)" test "$update_status" = 0 +check "No migration was skipped ($(a migrations_skipped) skipped)" test "$(a migrations_skipped)" = 0 +check "The system came back from the reboot by itself (came back: $rebooted)" test "$rebooted" = yes +check "It is a new boot (boot ID $(b boot_id) → $(a boot_id))" test -n "$(a boot_id)" -a "$(a boot_id)" != "$(b boot_id)" +check "It is on the $channel channel (got: $(a channel))" test "$(a channel)" = "$channel" +check "The running kernel is an installed one ($(a kernel))" test "$(a kernel_is_installed)" = yes +check "systemd reports the system running (got: $(a system_state))" test "$(a system_state)" = running +failed="$(a failed_units) $(a failed_user_units)" +check "No failed units (${failed// /}${failed:+ })" test -z "${failed// /}" +check "The desktop is up (Hyprland processes: $(a hyprland_running))" test "$(a hyprland_running)" = 1 +check "The wallpaper the user had still exists ($(a wallpaper))" test "$(a wallpaper_exists)" = yes +check "The boot menu has one Omarchy group ($(a boot_entry_groups))" test "$(a boot_entry_groups)" = 1 + +row() { printf '| %s | %s | %s |\n' "$1" "$(b "$2")" "$(a "$2")"; } +{ + if (( failures )); then echo "**$failures of ${#rows[@]} checks failed.**"; else echo "**All ${#rows[@]} checks passed.**"; fi + echo + echo "| | Before | After the update and a reboot |" + echo "| --- | --- | --- |" + row "Omarchy" omarchy + row "Channel" channel + row "Kernel running" kernel + row "systemd" systemd + row "Hyprland" hyprland + row "Migrations run" migrations_run + row "Wallpaper" wallpaper + echo + echo "The update took $update_seconds s and ran $migrations_in_update migrations." + echo + echo "| | Check |" + echo "| --- | --- |" + printf '%s\n' "${rows[@]}" +} >"$out/report.md" +cat "$out/report.md" +(( failures == 0 ))