From 7f0ae48661c45aa0865557fd733d366c764b4bfe Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Fri, 9 Oct 2026 23:52:50 +0200 Subject: [PATCH 1/3] Upgrade test: what an existing user gets from the next release Every test so far installs fresh. Nothing took a machine on the last release and updated it, which is what most users of a new release do. test/upgrade-test takes an installed system, moves it to edge or rc with omarchy-channel-set in a login shell, as a user would, reboots it through the firmware so the new kernel is the one that starts, and checks eleven things: the update's exit status, skipped migrations, that the reboot comes back by itself, into a new boot and an installed kernel, the channel, the system's state, failed units, the desktop, the wallpaper the user had, and the boot menu. It writes a report and takes a picture of the unlocked desktop before and after. A reboot that only comes back after a key press is a failure, and is said to be that. The Upgrade test workflow installs a release from its signed ISO with the install test, then runs it. It is started by hand, and runs for a pull request that changes the test. A second job, the only one that can write, commits the pictures to the upgrade-test-screenshots branch and posts the report as one comment on the branch's pull request, updated in place on later runs. Run by hand from 4.0.4 to today's edge it found two things: the wallpaper link left pointing at a file the update renamed (omacom/omarchy#14596), and one reboot in three stuck on a black screen until a key was pressed (omacom/omarchy#14771). --- .github/workflows/upgrade-test.yml | 227 +++++++++++++++++++++++++++++ README.md | 19 +++ test/upgrade-test | 221 ++++++++++++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100644 .github/workflows/upgrade-test.yml create mode 100755 test/upgrade-test diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml new file mode 100644 index 0000000..04c62f5 --- /dev/null +++ b/.github/workflows/upgrade-test.yml @@ -0,0 +1,227 @@ +name: Upgrade test + +# What an existing user gets: installs a release from its ISO, moves it to +# another channel the way a user would (omarchy-channel-set), reboots it, and +# checks what came back. test/upgrade-test does the work and says what it +# checks. +# +# Started by hand: from the latest release to edge unless told otherwise. It +# also runs for a pull request that changes the test itself. +# +# When there is a pull request for the branch it ran on, the result goes there +# as one comment, updated in place on later runs, with a picture of the desktop +# before and after. The pictures are committed to the upgrade-test-screenshots +# branch, which is what lets a comment show them. + +on: + workflow_dispatch: + inputs: + to: + description: "Channel to move to" + type: choice + options: [edge, rc] + default: edge + from: + description: "Release to start from, like 4.0.4 (default: the latest)" + type: string + default: "" + pull_request: + paths: + - test/upgrade-test + - .github/workflows/upgrade-test.yml + +permissions: + contents: read + +concurrency: + group: upgrade-test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + TO: ${{ inputs.to || 'edge' }} + +jobs: + upgrade: + name: Install, upgrade, reboot + # Our own branches run on Namespace; a pull request from a fork stays on + # GitHub's runner. + runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-26.04' || 'namespace-profile-linux-amd64-16x32' }} + timeout-minutes: 60 + outputs: + from: ${{ steps.release.outputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - id: release + env: + GH_TOKEN: ${{ github.token }} + FROM: ${{ inputs.from }} + run: | + set -euo pipefail + version=${FROM:-$(gh api repos/omacom/omarchy/releases/latest --jq .tag_name)} + version=${version#v} + [[ $version =~ ^[0-9]+(\.[0-9]+)+$ ]] || { echo "::error::'$version' is not a release version"; exit 1; } + echo "version=$version" | tee -a "$GITHUB_OUTPUT" + + # Good only if the signature is the release key's, the one in this repo. + - name: Download the released ISO and check its signature + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + mkdir -p release + iso=omarchy-$VERSION.iso + curl -fSL --retry 3 -o "release/$iso" "https://iso.omarchy.org/$iso" + curl -fsSL --retry 3 -o "release/$iso.sig" "https://iso.omarchy.org/$iso.sig" + home=$(mktemp -d) + gpg --homedir "$home" --quiet --batch --import builder/omarchy.gpg + key=$(gpg --homedir "$home" --batch --with-colons --list-keys | awk -F: '$1 == "fpr" { print $10; exit }') + gpg --homedir "$home" --batch --status-fd 1 --verify "release/$iso.sig" "release/$iso" 2>/dev/null | + grep -q "^\[GNUPG:\] VALIDSIG $key " || { echo "::error::$iso is not signed by $key"; exit 1; } + echo "$iso is signed by $key" + rm -rf "$home" "release/$iso.sig" + + # The install test's own install, which leaves the installed disk behind; + # the upgrade then starts from that disk. A failed upgrade does not stop + # here: the evidence is kept first, and the last step fails the job. + - name: Install the release, then upgrade it + run: | + sudo chmod 0666 /dev/kvm + docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \ + -e OMARCHY_INTEGRATION_DISK_CACHE=none -e TO \ + ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c ' + if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then + printf "#!/bin/sh\nexec /usr/bin/pacman --disable-sandbox \"\$@\"\n" >/usr/local/bin/pacman + chmod 0755 /usr/local/bin/pacman + fi + sed -i "1i Server = https://mirror.omarchy.org/\$repo/os/\$arch" /etc/pacman.d/mirrorlist + for attempt in 1 2 3; do + pacman -Syu --noconfirm --needed qemu-system-x86 qemu-hw-display-virtio-gpu qemu-hw-display-virtio-vga qemu-img edk2-ovmf socat \ + imagemagick tesseract tesseract-data-eng mtools dosfstools jq openssh openssl python curl >/dev/null && break + [ "$attempt" -lt 3 ] || { echo "::error::the test container could not install its tools"; exit 1; } + echo "pacman failed, attempt $attempt of 3; trying again in 15 s"; sleep 15 + done + printf "#!/bin/sh\nexit 0\n" >/usr/local/bin/omarchy-pkg-add + chmod 0755 /usr/local/bin/omarchy-pkg-add + ./test/integration release/*.iso boot --no-preview --memory 8192 --timeout 1800 + + status=0 + VM_STATE_DIR=/w ./test/upgrade-test "$(ls -d test-runs/*-integration)" "$TO" upgrade-evidence || status=$? + echo "$status" >upgrade-evidence/status + ' + + - name: Hand the results to the runner's user + if: always() + run: sudo chown -R "$(id -u):$(id -g)" upgrade-evidence test-runs 2>/dev/null || true + + - name: Keep the evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} + path: upgrade-evidence/ + if-no-files-found: ignore + retention-days: 14 + + - name: The upgrade's result + run: | + cat upgrade-evidence/report.md >>"$GITHUB_STEP_SUMMARY" + [ "$(cat upgrade-evidence/status)" = 0 ] || { echo "::error::the upgrade test failed, see the summary"; exit 1; } + + # The only job that can write: it commits the two pictures and writes the + # comment. It runs nothing from the test; it handles two PNG files and a + # Markdown report. + report: + name: Report + needs: upgrade + if: ${{ !cancelled() && !github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + env: + GH_TOKEN: ${{ github.token }} + FROM: ${{ needs.upgrade.outputs.from }} + PR: ${{ github.event.pull_request.number }} + BRANCH: ${{ github.head_ref || github.ref_name }} + SHOTS_BRANCH: upgrade-test-screenshots + steps: + # No evidence at all still gets a comment saying so. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + continue-on-error: true + with: + name: upgrade-test-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + + # A comment can only show a picture that has a public address. A commit + # on a branch of this repository gives it one, and the address of a + # commit never changes. + - name: Commit the pictures + id: pictures + run: | + set -euo pipefail + [ -s evidence/before.png ] || [ -s evidence/after.png ] || { echo "no pictures to commit"; exit 0; } + url=https://x-access-token:$GH_TOKEN@github.com/$GITHUB_REPOSITORY.git + git init -q shots && cd shots + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git fetch -q --depth 1 "$url" "$SHOTS_BRANCH" 2>/dev/null; then + git checkout -q -b "$SHOTS_BRANCH" FETCH_HEAD + else + git checkout -q --orphan "$SHOTS_BRANCH" + fi + dir=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT + mkdir -p "$dir" + cp ../evidence/*.png "$dir/" + git add "$dir" + git commit -q -m "Upgrade test $GITHUB_RUN_ID, attempt $GITHUB_RUN_ATTEMPT: $FROM to $TO" + git push -q "$url" "$SHOTS_BRANCH" + echo "base=https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$(git rev-parse HEAD)/$dir" >>"$GITHUB_OUTPUT" + + - name: Write the comment + env: + PICTURES: ${{ steps.pictures.outputs.base }} + RESULT: ${{ needs.upgrade.result }} + run: | + set -euo pipefail + marker="" + { + echo "$marker" + echo "## Upgrade test: $FROM → $TO" + echo + if [ -s evidence/report.md ]; then + cat evidence/report.md + else + echo "**The test did not get as far as a report** (job result: $RESULT)." + fi + if [ -n "$PICTURES" ]; then + echo + echo "| Before, $FROM | After, on $TO |" + echo "| --- | --- |" + echo "| ![before]($PICTURES/before.png) | ![after]($PICTURES/after.png) |" + if [ -s evidence/stuck.png ]; then + echo + echo "The reboot did not come back by itself. This is what the screen showed:" + echo + echo "![stuck]($PICTURES/stuck.png)" + fi + fi + echo + echo "[Run $GITHUB_RUN_ID]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID), attempt $GITHUB_RUN_ATTEMPT, at \`${GITHUB_SHA:0:7}\`." + } >comment.md + cat comment.md >>"$GITHUB_STEP_SUMMARY" + + # A run started by hand has no pull request of its own: use the open + # one for its branch, if there is one. + pr=${PR:-$(gh pr list -R "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq '.[0].number // empty')} + [ -n "$pr" ] || { echo "No pull request for $BRANCH: the result is on this run's page."; exit 0; } + existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$pr/comments" --paginate \ + --jq "[.[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))][0].id // empty") + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@comment.md >/dev/null + echo "Updated the comment on #$pr" + else + gh api "repos/$GITHUB_REPOSITORY/issues/$pr/comments" -F body=@comment.md >/dev/null + echo "Commented on #$pr" + fi diff --git a/README.md b/README.md index cc5fe51..e82eedd 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,25 @@ The first scenario is `factory-reset`: it proves `omarchy-system-factory-reset` Artifacts — screenshots, the fixtured/staged/final `limine.conf`, the reset typescript, and the factory-reset log — land under `test-runs/-integration/runs/-/`, and `--no-preview` skips the `imv` review just like the acceptance harness. +## Upgrade testing + +`test/upgrade-test` checks what an existing user gets. It takes an installed system, moves it to another channel with `omarchy-channel-set`, reboots it, and checks what came back: the update's exit status, skipped migrations, that the reboot comes back by itself into an installed kernel, failed units, the desktop, the user's wallpaper and the boot menu. + +```bash +./test/integration release/omarchy.iso boot # installs once, leaves the installed disk +./test/upgrade-test test-runs/omarchy-integration edge upgrade-evidence # edge or rc +``` + +The evidence folder gets `report.md`, the update's output, and a picture of the desktop before and after. + +The "Upgrade test" workflow does both steps from a release's ISO. Start it by hand, from the latest release to edge by default: + +```bash +gh workflow run upgrade-test.yml -f to=edge +``` + +When the branch it ran on has a pull request, the result is posted there as one comment and updated in place on later runs. + ## Signing the ISO Run `./bin/omarchy-iso-sign [release/omarchy.iso]`. The signing key is retrieved from the shared Omarchy vault with the 1Password CLI. diff --git a/test/upgrade-test b/test/upgrade-test new file mode 100755 index 0000000..1df9e49 --- /dev/null +++ b/test/upgrade-test @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Takes an installed Omarchy, as a user's machine holds it, moves it to another +# channel the way a user would, reboots it, and checks what came back. +# +# test/upgrade-test +# +# installed_dir is what test/integration leaves once its install has passed, +# test-runs/-integration: base.qcow2, OVMF_VARS.4m.fd and id_ed25519. It is +# not written to; the machine runs on an overlay. channel is edge or rc. +# +# The machine boots through the firmware and the boot loader, so the reboot +# after the update starts whatever kernel the update installed. Commands run +# in a login shell, as a user's terminal does: Omarchy's commands need its +# environment. +# +# evidence_dir gets before.png and after.png (the desktop, unlocked), the +# update's output, the facts before and after, and report.md; stuck.png too if +# the reboot did not come back by itself. The exit status is 0 only if every +# check passed. +# +# Needs qemu-system-x86_64, qemu-img, ssh, python3 and a usable /dev/kvm. +# Env: SMP (4), MEM (8G), SSH_PORT (2422), GUEST_PASSWORD (omarchy), +# UPDATE_TIMEOUT (seconds, 1800), EXTRA_MIGRATIONS (migration files to put +# in place before the update, to try a fix that is not released yet). +set -uo pipefail +[[ $# -eq 3 ]] || { sed -n '3,24p' "$0" >&2; exit 2; } +in=$(realpath "$1") channel=$2 out=$(realpath -m "$3") +pw=${GUEST_PASSWORD:-omarchy} port=${SSH_PORT:-2422} +[[ $channel == edge || $channel == rc ]] || { echo "upgrade-test: channel is edge or rc" >&2; exit 2; } +for f in base.qcow2 OVMF_VARS.4m.fd id_ed25519; do + [[ -s $in/$f ]] || { echo "upgrade-test: $in/$f is missing" >&2; exit 1; } +done +[[ -r /dev/kvm && -w /dev/kvm ]] || { echo "upgrade-test: /dev/kvm is not usable" >&2; exit 1; } +for fw in /usr/share/edk2/x64/OVMF_CODE.4m.fd /usr/share/OVMF/OVMF_CODE_4M.fd; do + [[ -f $fw ]] && { fw_code=$fw; break; } +done +: "${fw_code:?no UEFI firmware found}" + +mkdir -p "$out" +state=$(mktemp -d "${VM_STATE_DIR:-/var/tmp}/upgrade-test.XXXXXX") +cleanup() { + local status=$? pid + if [[ -f $state/qemu.pid ]]; then + pid=$(cat "$state/qemu.pid") + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 50); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done + fi + rm -rf "$state" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +log() { printf '==> %s\n' "$*"; } +install -m600 "$in/id_ed25519" "$state/key" +ssh_opts=(-i "$state/key" -p "$port" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=3) +guest() { ssh "${ssh_opts[@]}" omarchy@127.0.0.1 "bash -lc $(printf %q "$1")" /dev/null; do + kill -0 "$(cat "$state/qemu.pid")" 2>/dev/null || { echo "upgrade-test: qemu exited" >&2; return 1; } + (( SECONDS - start < $1 )) || return 1 + sleep 1 + done +} +qmp() { # shot | key ... + python3 - "$state/qmp.sock" "$@" <<'PY' +import json, socket, sys +s = socket.socket(socket.AF_UNIX); s.settimeout(20); s.connect(sys.argv[1]); f = s.makefile("rw"); f.readline() +def run(command): + f.write(json.dumps(command) + "\n"); f.flush() + while "return" not in (reply := json.loads(f.readline())): + if "error" in reply: sys.exit(reply["error"]["desc"]) +run({"execute": "qmp_capabilities"}) +if sys.argv[2] == "shot": + run({"execute": "screendump", "arguments": {"filename": sys.argv[3], "format": "png"}}) +else: + for key in sys.argv[3:]: + run({"execute": "send-key", "arguments": {"keys": [{"type": "qcode", "data": key}]}}) +PY +} +# The session starts locked. Type the password, as its owner would, and wait +# for the desktop to draw. +desktop_shot() { # file + local key keys=() + for (( key = 0; key < ${#pw}; key++ )); do keys+=("${pw:key:1}"); done + qmp key "${keys[@]}" ret || true + sleep 8 + qmp shot "$1" || echo "upgrade-test: no screenshot" >&2 +} +# One fact per line, name=value, the same names before and after. +facts() { # file + guest ' + link=~/.local/state/omarchy/current/background; target=$(readlink -f "$link" 2>/dev/null) + echo "omarchy=$(pacman -Qq omarchy omarchy-dev 2>/dev/null | sort -u | xargs -r pacman -Q 2>/dev/null | paste -sd, -)" + echo "channel=$(omarchy-channel-current 2>/dev/null)" + echo "kernel=$(uname -r)" + echo "kernel_is_installed=$([[ -d /usr/lib/modules/$(uname -r)/kernel ]] && echo yes || echo no)" + echo "systemd=$(systemctl --version | sed -n "1s/^systemd \([0-9.]*\).*/\1/p")" + echo "hyprland=$(pacman -Q hyprland 2>/dev/null | cut -d" " -f2)" + echo "system_state=$(systemctl is-system-running 2>/dev/null)" + echo "failed_units=$(systemctl --failed --no-legend --plain | awk "{print \$1}" | paste -sd, -)" + echo "failed_user_units=$(systemctl --user --failed --no-legend --plain | awk "{print \$1}" | paste -sd, -)" + echo "hyprland_running=$(pgrep -c -x Hyprland)" + echo "migrations_run=$(find ~/.local/state/omarchy/migrations -maxdepth 1 -name "*.sh" | wc -l)" + echo "migrations_skipped=$(find ~/.local/state/omarchy/migrations/skipped -name "*.sh" 2>/dev/null | wc -l)" + echo "wallpaper=${target##*/}" + echo "wallpaper_exists=$([[ -n $target && -e $target ]] && echo yes || echo no)" + echo "boot_entry_groups=$(sudo grep -c "^/+" /boot/limine.conf 2>/dev/null)" + echo "boot_id=$(cut -c1-8 /proc/sys/kernel/random/boot_id)" + ' 2>/dev/null | grep -E '^[a-z_]+=' >"$1" +} +fact() { sed -n "s/^$2=//p" "$1" | head -n 1; } + +qemu-img create -qf qcow2 -b "$in/base.qcow2" -F qcow2 "$state/run.qcow2" +cp "$in/OVMF_VARS.4m.fd" "$state/vars.fd" +: >"$out/serial.log" +qemu-system-x86_64 -cpu host -enable-kvm -machine q35,accel=kvm -smp "${SMP:-4}" -m "${MEM:-8G}" \ + -drive "if=pflash,format=raw,readonly=on,file=$fw_code" -drive "if=pflash,format=raw,file=$state/vars.fd" \ + -drive "file=$state/run.qcow2,format=qcow2,if=none,id=d0" -device virtio-blk-pci,drive=d0 \ + -device virtio-vga -device virtio-rng-pci -display none -usb -device usb-kbd -serial "file:$out/serial.log" \ + -netdev "user,id=n0,hostfwd=tcp:127.0.0.1:$port-:22" -device virtio-net-pci,netdev=n0 \ + -qmp "unix:$state/qmp.sock,server,nowait" -pidfile "$state/qemu.pid" -daemonize + +log "Booting the installed system" +wait_ssh 300 || { echo "upgrade-test: the installed system did not come up" >&2; exit 1; } +# The overlay is thrown away: let sudo through without asking, so a long +# update does not stop at a password prompt nobody is there to answer. +ssh "${ssh_opts[@]}" omarchy@127.0.0.1 "echo $(printf %q "$pw") | sudo -S -p '' sh -c 'echo \"omarchy ALL=(ALL) NOPASSWD: ALL\" >/etc/sudoers.d/99-upgrade-test'" /dev/null 2>&1 || true' +facts "$out/before.txt" +desktop_shot "$out/before.png" +log "Before: $(fact "$out/before.txt" omarchy), kernel $(fact "$out/before.txt" kernel), channel $(fact "$out/before.txt" channel)" + +for migration in ${EXTRA_MIGRATIONS:-}; do + log "Putting $(basename "$migration") in place before the update" + ssh "${ssh_opts[@]}" omarchy@127.0.0.1 "sudo tee /usr/share/omarchy/migrations/$(basename "$migration") >/dev/null" <"$migration" +done + +log "omarchy-channel-set $channel" +update_start=$SECONDS +guest "timeout ${UPDATE_TIMEOUT:-1800} omarchy-channel-set $channel" >"$out/update.log" 2>&1 +update_status=$? +update_seconds=$(( SECONDS - update_start )) +sed -i 's/\x1b\[[0-9;?]*[A-Za-z]//g; s/\r/\n/g' "$out/update.log" +migrations_in_update=$(grep -c '^Running migration' "$out/update.log" || true) +log "The update exited $update_status after $update_seconds s, $migrations_in_update migrations" + +log "Rebooting" +guest 'sudo systemctl reboot' >/dev/null 2>&1 || true +# Wait for the old boot to go away first: its sshd answers for a moment yet. +for _ in $(seq 1 90); do + ssh "${ssh_opts[@]}" -o ConnectTimeout=2 omarchy@127.0.0.1 true /dev/null || break + sleep 1 +done +rebooted=no +if wait_ssh 150; then + rebooted=yes +else + # A boot that sits there until a key is pressed is a failure, and a known + # kind: keep what the screen showed, then press Enter to find out whether + # that was it, and to have a system to collect the rest from. + qmp shot "$out/stuck.png" || true + qmp key ret || true + if wait_ssh 150; then rebooted="only after a key was pressed"; fi +fi +if [[ $rebooted != no ]]; then + guest 'timeout 180 systemctl is-system-running --wait >/dev/null 2>&1 || true' + facts "$out/after.txt" + desktop_shot "$out/after.png" +else + : >"$out/after.txt" + qmp shot "$out/after.png" || true +fi +log "After: $(fact "$out/after.txt" omarchy), kernel $(fact "$out/after.txt" kernel), channel $(fact "$out/after.txt" channel)" + +failures=0 +rows=() +check() { # description, command... + local what=$1; shift + if "$@"; then rows+=("| ✅ | $what |"); else rows+=("| ❌ | $what |"); failures=$((failures + 1)); fi +} +b() { fact "$out/before.txt" "$1"; } +a() { fact "$out/after.txt" "$1"; } +check "The update finished without an error (exit $update_status)" test "$update_status" = 0 +check "No migration was skipped ($(a migrations_skipped) skipped)" test "$(a migrations_skipped)" = 0 +check "The system came back from the reboot by itself (came back: $rebooted)" test "$rebooted" = yes +check "It is a new boot (boot ID $(b boot_id) → $(a boot_id))" test -n "$(a boot_id)" -a "$(a boot_id)" != "$(b boot_id)" +check "It is on the $channel channel (got: $(a channel))" test "$(a channel)" = "$channel" +check "The running kernel is an installed one ($(a kernel))" test "$(a kernel_is_installed)" = yes +check "systemd reports the system running (got: $(a system_state))" test "$(a system_state)" = running +failed="$(a failed_units) $(a failed_user_units)" +check "No failed units (${failed// /}${failed:+ })" test -z "${failed// /}" +check "The desktop is up (Hyprland processes: $(a hyprland_running))" test "$(a hyprland_running)" = 1 +check "The wallpaper the user had still exists ($(a wallpaper))" test "$(a wallpaper_exists)" = yes +check "The boot menu has one Omarchy group ($(a boot_entry_groups))" test "$(a boot_entry_groups)" = 1 + +row() { printf '| %s | %s | %s |\n' "$1" "$(b "$2")" "$(a "$2")"; } +{ + if (( failures )); then echo "**$failures of ${#rows[@]} checks failed.**"; else echo "**All ${#rows[@]} checks passed.**"; fi + echo + echo "| | Before | After the update and a reboot |" + echo "| --- | --- | --- |" + row "Omarchy" omarchy + row "Channel" channel + row "Kernel running" kernel + row "systemd" systemd + row "Hyprland" hyprland + row "Migrations run" migrations_run + row "Wallpaper" wallpaper + echo + echo "The update took $update_seconds s and ran $migrations_in_update migrations." + echo + echo "| | Check |" + echo "| --- | --- |" + printf '%s\n' "${rows[@]}" +} >"$out/report.md" +cat "$out/report.md" +(( failures == 0 )) From b5936a6cbebab508ac8ede77787e3e9fceef4962 Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Sat, 10 Oct 2026 00:09:56 +0200 Subject: [PATCH 2/3] Upgrade test: started by hand only, and two checks that prove what they say The workflow no longer runs for pull requests. It is started by hand, and still posts to the pull request of the branch it was started on. The fork fallback and the fork guard go with the trigger. Two checks could pass without the thing they named being true: The wallpaper check asked only whether the file the link points at exists, so an update that swapped the user's picture for another would pass. It now compares before and after: the same picture, whatever format it is kept in, and present. The failed-units check read an empty answer as nothing failed, which is also what a service manager that cannot be asked gives. A query that fails is now recorded as that, and fails the check. --- .github/workflows/upgrade-test.yml | 23 +++++++---------------- README.md | 2 +- test/upgrade-test | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index 04c62f5..0413746 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -5,8 +5,7 @@ name: Upgrade test # checks what came back. test/upgrade-test does the work and says what it # checks. # -# Started by hand: from the latest release to edge unless told otherwise. It -# also runs for a pull request that changes the test itself. +# Started by hand only: from the latest release to edge unless told otherwise. # # When there is a pull request for the branch it ran on, the result goes there # as one comment, updated in place on later runs, with a picture of the desktop @@ -25,16 +24,12 @@ on: description: "Release to start from, like 4.0.4 (default: the latest)" type: string default: "" - pull_request: - paths: - - test/upgrade-test - - .github/workflows/upgrade-test.yml permissions: contents: read concurrency: - group: upgrade-test-${{ github.event.pull_request.number || github.ref }} + group: upgrade-test-${{ github.ref }} cancel-in-progress: true env: @@ -43,9 +38,7 @@ env: jobs: upgrade: name: Install, upgrade, reboot - # Our own branches run on Namespace; a pull request from a fork stays on - # GitHub's runner. - runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-26.04' || 'namespace-profile-linux-amd64-16x32' }} + runs-on: namespace-profile-linux-amd64-16x32 timeout-minutes: 60 outputs: from: ${{ steps.release.outputs.version }} @@ -134,7 +127,7 @@ jobs: report: name: Report needs: upgrade - if: ${{ !cancelled() && !github.event.pull_request.head.repo.fork }} + if: ${{ !cancelled() }} runs-on: ubuntu-26.04 timeout-minutes: 10 permissions: @@ -143,8 +136,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} FROM: ${{ needs.upgrade.outputs.from }} - PR: ${{ github.event.pull_request.number }} - BRANCH: ${{ github.head_ref || github.ref_name }} + BRANCH: ${{ github.ref_name }} SHOTS_BRANCH: upgrade-test-screenshots steps: # No evidence at all still gets a comment saying so. @@ -212,9 +204,8 @@ jobs: } >comment.md cat comment.md >>"$GITHUB_STEP_SUMMARY" - # A run started by hand has no pull request of its own: use the open - # one for its branch, if there is one. - pr=${PR:-$(gh pr list -R "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq '.[0].number // empty')} + # The open pull request for the branch this was started on, if any. + pr=$(gh pr list -R "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq '.[0].number // empty') [ -n "$pr" ] || { echo "No pull request for $BRANCH: the result is on this run's page."; exit 0; } existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$pr/comments" --paginate \ --jq "[.[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))][0].id // empty") diff --git a/README.md b/README.md index e82eedd..6ac625c 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ The "Upgrade test" workflow does both steps from a release's ISO. Start it by ha gh workflow run upgrade-test.yml -f to=edge ``` -When the branch it ran on has a pull request, the result is posted there as one comment and updated in place on later runs. +It only runs when started. When the branch it ran on has a pull request, the result is posted there as one comment and updated in place on later runs. ## Signing the ISO diff --git a/test/upgrade-test b/test/upgrade-test index 1df9e49..5f4976e 100755 --- a/test/upgrade-test +++ b/test/upgrade-test @@ -101,8 +101,11 @@ facts() { # file echo "systemd=$(systemctl --version | sed -n "1s/^systemd \([0-9.]*\).*/\1/p")" echo "hyprland=$(pacman -Q hyprland 2>/dev/null | cut -d" " -f2)" echo "system_state=$(systemctl is-system-running 2>/dev/null)" - echo "failed_units=$(systemctl --failed --no-legend --plain | awk "{print \$1}" | paste -sd, -)" - echo "failed_user_units=$(systemctl --user --failed --no-legend --plain | awk "{print \$1}" | paste -sd, -)" + # A manager that cannot be asked is not a manager with nothing failed. + units=$(systemctl --failed --no-legend --plain 2>/dev/null) && units=$(awk "{print \$1}" <<<"$units" | paste -sd, -) || units="could not ask systemd" + user_units=$(systemctl --user --failed --no-legend --plain 2>/dev/null) && user_units=$(awk "{print \$1}" <<<"$user_units" | paste -sd, -) || user_units="could not ask the user manager" + echo "failed_units=$units" + echo "failed_user_units=$user_units" echo "hyprland_running=$(pgrep -c -x Hyprland)" echo "migrations_run=$(find ~/.local/state/omarchy/migrations -maxdepth 1 -name "*.sh" | wc -l)" echo "migrations_skipped=$(find ~/.local/state/omarchy/migrations/skipped -name "*.sh" 2>/dev/null | wc -l)" @@ -192,9 +195,14 @@ check "It is on the $channel channel (got: $(a channel))" test "$(a channel)" = check "The running kernel is an installed one ($(a kernel))" test "$(a kernel_is_installed)" = yes check "systemd reports the system running (got: $(a system_state))" test "$(a system_state)" = running failed="$(a failed_units) $(a failed_user_units)" -check "No failed units (${failed// /}${failed:+ })" test -z "${failed// /}" +failed=$(xargs <<<"$failed") +check "No failed units (${failed:-none})" test -z "$failed" check "The desktop is up (Hyprland processes: $(a hyprland_running))" test "$(a hyprland_running)" = 1 -check "The wallpaper the user had still exists ($(a wallpaper))" test "$(a wallpaper_exists)" = yes +# The same picture, whatever format it is kept in now: an update may convert +# the bundled images, and may not swap the one the user chose for another. +before_wallpaper=$(b wallpaper) after_wallpaper=$(a wallpaper) +check "The wallpaper is the one the user had, and exists ($before_wallpaper → $after_wallpaper)" \ + test "$(a wallpaper_exists)" = yes -a -n "$after_wallpaper" -a "${after_wallpaper%.*}" = "${before_wallpaper%.*}" check "The boot menu has one Omarchy group ($(a boot_entry_groups))" test "$(a boot_entry_groups)" = 1 row() { printf '| %s | %s | %s |\n' "$1" "$(b "$2")" "$(a "$2")"; } From d6a32f0dcf39b3599d5d6a6733f1e3429a6bb135 Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Sat, 10 Oct 2026 00:20:27 +0200 Subject: [PATCH 3/3] Upgrade test: fail when nothing was upgraded, and retry a lost push A channel that holds the version the machine already has upgrades nothing, and every other check passes on a machine that was only rebooted: rc is in that state today. A twelfth check requires the Omarchy package to have changed. Two runs on different branches both push pictures to one branch. The second to push was refused and its comment never written. The push is now tried again on top of whatever got there first; each run's folder is its own, so there is nothing to merge. --- .github/workflows/upgrade-test.yml | 30 +++++++++++++++++++----------- test/upgrade-test | 3 +++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index 0413746..800f5da 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -155,20 +155,28 @@ jobs: set -euo pipefail [ -s evidence/before.png ] || [ -s evidence/after.png ] || { echo "no pictures to commit"; exit 0; } url=https://x-access-token:$GH_TOKEN@github.com/$GITHUB_REPOSITORY.git + dir=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT git init -q shots && cd shots git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - if git fetch -q --depth 1 "$url" "$SHOTS_BRANCH" 2>/dev/null; then - git checkout -q -b "$SHOTS_BRANCH" FETCH_HEAD - else - git checkout -q --orphan "$SHOTS_BRANCH" - fi - dir=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT - mkdir -p "$dir" - cp ../evidence/*.png "$dir/" - git add "$dir" - git commit -q -m "Upgrade test $GITHUB_RUN_ID, attempt $GITHUB_RUN_ATTEMPT: $FROM to $TO" - git push -q "$url" "$SHOTS_BRANCH" + # Runs on other branches push here too. A push that loses the race + # is made again on top of whatever won: the folder is this run's own, + # so there is never anything to merge. + for attempt in 1 2 3 4 5; do + if git fetch -q --depth 1 "$url" "$SHOTS_BRANCH" 2>/dev/null; then + git checkout -q -B "$SHOTS_BRANCH" FETCH_HEAD + else + git checkout -q --orphan "$SHOTS_BRANCH" 2>/dev/null || true + fi + mkdir -p "$dir" + cp ../evidence/*.png "$dir/" + git add "$dir" + git commit -q -m "Upgrade test $GITHUB_RUN_ID, attempt $GITHUB_RUN_ATTEMPT: $FROM to $TO" + if git push -q "$url" "HEAD:refs/heads/$SHOTS_BRANCH"; then break; fi + [ "$attempt" -lt 5 ] || { echo "::error::could not push the pictures after 5 tries"; exit 1; } + echo "The push lost a race, trying again" + sleep $((attempt * 2)) + done echo "base=https://raw.githubusercontent.com/$GITHUB_REPOSITORY/$(git rev-parse HEAD)/$dir" >>"$GITHUB_OUTPUT" - name: Write the comment diff --git a/test/upgrade-test b/test/upgrade-test index 5f4976e..2bd9c76 100755 --- a/test/upgrade-test +++ b/test/upgrade-test @@ -192,6 +192,9 @@ check "No migration was skipped ($(a migrations_skipped) skipped)" test "$(a mig check "The system came back from the reboot by itself (came back: $rebooted)" test "$rebooted" = yes check "It is a new boot (boot ID $(b boot_id) → $(a boot_id))" test -n "$(a boot_id)" -a "$(a boot_id)" != "$(b boot_id)" check "It is on the $channel channel (got: $(a channel))" test "$(a channel)" = "$channel" +# A channel that holds what the machine already had upgrades nothing, and +# every other check would pass on a machine that was only rebooted. +check "Omarchy itself changed ($(b omarchy) → $(a omarchy))" test -n "$(a omarchy)" -a "$(a omarchy)" != "$(b omarchy)" check "The running kernel is an installed one ($(a kernel))" test "$(a kernel_is_installed)" = yes check "systemd reports the system running (got: $(a system_state))" test "$(a system_state)" = running failed="$(a failed_units) $(a failed_user_units)"