From f9caa2576672f1bcb3b9399f548c46e73da19f82 Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Fri, 9 Oct 2026 22:32:58 +0200 Subject: [PATCH 1/4] Nightly: a current rclone for the upload, and a warning when it retries Ubuntu's rclone is 1.60.1, from 2022. In the first publish R2 answered every upload of the first attempt "501 Not Implemented", having stored the file each time, and the job passed because rclone's second attempt found everything in place. Run against a stand-in server, 1.60.1 and the current 1.75.2 make the same requests but for one header: 1.60.1 sends "x-amz-acl: private" with each upload and 1.75.2 sends none to Cloudflare. The publish job now installs 1.75.2, by version and checksum. Should an upload pass only on a retry again, the run says so in a warning. --- .github/workflows/nightly-build.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index a54a7ae..30209f8 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -262,18 +262,36 @@ jobs: sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null ./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out + # A current rclone, by version and checksum. Ubuntu's is 1.60 from 2022: + # with it R2 answered every first upload "501 Not Implemented", though + # it had stored the file, and the publish only passed on rclone's retry. + # 1.60 sends "x-amz-acl: private" with each upload; current rclone does + # not send it to Cloudflare. - name: rclone + env: + RCLONE_VERSION: v1.75.2 + RCLONE_SHA256: efbfe852181f7191eb3c9043ed1ab49c9b2d0ba045c61c926a5da111c939ec5c run: | + set -euo pipefail [ -n "$RCLONE_CONFIG_R2_ACCESS_KEY_ID" ] || { echo "::error::the nightly environment has no bucket key"; exit 1; } - sudo apt-get update -qq && sudo apt-get install -y -qq rclone >/dev/null + curl -fsSL --retry 3 -o rclone.deb "https://github.com/rclone/rclone/releases/download/$RCLONE_VERSION/rclone-$RCLONE_VERSION-linux-amd64.deb" + echo "$RCLONE_SHA256 rclone.deb" | sha256sum -c - + sudo dpkg -i rclone.deb >/dev/null rclone version | head -1 # --immutable: a name that exists with other bytes is an error, not an # overwrite. + # An upload that only passes on a retry is said so on the run's page: + # the files are compared afterwards either way, but it should not go + # unnoticed again. - name: Upload this version run: | + set -euo pipefail rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \ - --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line + --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line 2>&1 | tee upload.log + if grep -q "Failed to copy" upload.log; then + echo "::warning::rclone needed a retry: $(grep -c "Failed to copy" upload.log) uploads failed on an earlier attempt" + fi # Fetched the way a user fetches them. It proves what is served, and it # is the first request the CDN sees for each file. From 22ef6e12c5bb27ad83cef8884cba0752bedc3dad Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Fri, 9 Oct 2026 22:35:17 +0200 Subject: [PATCH 2/4] CI: GitHub's runners on Ubuntu 26.04 The four jobs on GitHub's own runners move from ubuntu-24.04 and ubuntu-latest to ubuntu-26.04. On 26.04 /tmp is held in memory, so the publish job's artifact, 11 GB for edge, goes to the runner's temp directory on disk instead. nightly-fetch now checks there is room before it starts and says how much is missing. --- .github/workflows/install-test-comment.yml | 2 +- .github/workflows/install-test.yml | 4 ++-- .github/workflows/nightly-build.yml | 10 +++++++--- .github/workflows/unit-test.yml | 2 +- test/unit/nightly-test.sh | 2 ++ test/vm-image/nightly-fetch | 11 +++++++++-- 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/install-test-comment.yml b/.github/workflows/install-test-comment.yml index 3820c3c..b4c9c7b 100644 --- a/.github/workflows/install-test-comment.yml +++ b/.github/workflows/install-test-comment.yml @@ -18,7 +18,7 @@ permissions: jobs: comment: if: github.event.workflow_run.event == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} diff --git a/.github/workflows/install-test.yml b/.github/workflows/install-test.yml index ebd55ed..1e8470e 100644 --- a/.github/workflows/install-test.yml +++ b/.github/workflows/install-test.yml @@ -25,7 +25,7 @@ jobs: # GitHub's runner: it is somebody else's code, and it has no business on # our account's machines. No cache volume either way: what one pull # request leaves in a cache the next run would trust. - runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-24.04' || 'namespace-profile-linux-amd64-16x32' }} + runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-26.04' || 'namespace-profile-linux-amd64-16x32' }} timeout-minutes: 75 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -43,7 +43,7 @@ jobs: run: NO_BOOT_OFFER=1 ./bin/omarchy-iso-make # The harness runs in an Arch container: Arch's current QEMU and OVMF - # (QEMU 11) instead of Ubuntu 24.04's QEMU 8.2, at the paths the + # (QEMU 11) instead of the runner's older one, at the paths the # harness expects, the same as on an Omarchy workstation. - name: Install unattended, then boot the installed system run: | diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 30209f8..d109201 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -50,7 +50,7 @@ jobs: # test/vm-image/nightly-plan decides, from what is fetched here. plan: name: Plan - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 5 outputs: channels: ${{ steps.plan.outputs.channels }} @@ -229,7 +229,7 @@ jobs: fail-fast: false matrix: include: ${{ fromJSON(needs.plan.outputs.channels) }} - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 45 environment: nightly # actions: read is for finding and fetching the build job's artifact. @@ -258,8 +258,12 @@ jobs: - name: Fetch the release from the build job env: GH_TOKEN: ${{ github.token }} + # Not /tmp: on Ubuntu 26.04 it is held in memory, and the zip is + # gigabytes. + TMPDIR: ${{ runner.temp }} run: | - sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null + df -h "$TMPDIR" . + command -v aria2c unzip >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null; } ./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out # A current rclone, by version and checksum. Ubuntu's is 1.60 from 2022: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 823b6bd..137346f 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -15,7 +15,7 @@ permissions: jobs: unit: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/test/unit/nightly-test.sh b/test/unit/nightly-test.sh index cf7624d..f17bd5d 100644 --- a/test/unit/nightly-test.sh +++ b/test/unit/nightly-test.sh @@ -181,6 +181,8 @@ STUB refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3" refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4" [[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked" + refused "an artifact larger than the free space" env FETCH_SIZE=9000000000000000 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f5" + grep -q "MiB free" "$tmp/refused.err" || fail "fetch: too little room is not what was reported" else echo "nightly-fetch tests skipped: no zip or unzip here" fi diff --git a/test/vm-image/nightly-fetch b/test/vm-image/nightly-fetch index 723c4a5..0ce1f04 100755 --- a/test/vm-image/nightly-fetch +++ b/test/vm-image/nightly-fetch @@ -8,11 +8,14 @@ # release would outlast the job. An artifact is one zip at a signed address, # so this asks the API for that address and takes it over 16 connections. # +# The zip goes to TMPDIR and is deleted once unpacked, so there has to be room +# for the artifact twice: point TMPDIR at a disk, not at a /tmp held in memory. +# # Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c # and unzip. A run started again keeps the artifacts of its earlier attempts # under the same name: the newest is the one fetched. set -euo pipefail -[[ $# -eq 4 ]] || { sed -n '2,13p' "$0" >&2; exit 2; } +[[ $# -eq 4 ]] || { sed -n '2,16p' "$0" >&2; exit 2; } repo=$1 run=$2 name=$3 out=$4 fail() { echo "nightly-fetch: $*" >&2; exit 1; } @@ -23,6 +26,11 @@ size=$(gh api "repos/$repo/actions/artifacts/$id" --jq .size_in_bytes) echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB" zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip") +mkdir -p "$out" +for place in "$(dirname "$zip")" "$out"; do + free=$(( $(df --output=avail -B1 "$place" | tail -n 1) )) + (( free > size )) || fail "$place has $((free / 1048576)) MiB free, the artifact is $((size / 1048576)) MiB" +done trap 'rm -f "$zip" "$zip.aria2"' EXIT started=$(date +%s) # The address is signed for a few minutes. A try that fails asks for a new one @@ -44,6 +52,5 @@ done seconds=$(( $(date +%s) - started )) echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s" -mkdir -p "$out" unzip -q "$zip" -d "$out" echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out" From ed0dfc50529b78f6e32c98b96c2f43299d9f6e4d Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Fri, 9 Oct 2026 22:36:31 +0200 Subject: [PATCH 3/4] Nightly: the retry warning covers latest.json and counts files The warning was written for the bulk upload only, and counted lines: latest.json is uploaded by a second rclone call, which could still pass on a retry unseen, and a file that failed on two attempts was counted as two uploads. Both calls go through test/vm-image/nightly-rclone, which passes rclone's output and exit status through and names each file once. A stand-in rclone tests it. --- .github/workflows/nightly-build.yml | 16 ++++++---------- test/unit/nightly-test.sh | 27 +++++++++++++++++++++++++++ test/vm-image/nightly-rclone | 19 +++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100755 test/vm-image/nightly-rclone diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index d109201..5bd0333 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -285,17 +285,13 @@ jobs: # --immutable: a name that exists with other bytes is an error, not an # overwrite. - # An upload that only passes on a retry is said so on the run's page: - # the files are compared afterwards either way, but it should not go - # unnoticed again. + # nightly-rclone is rclone, with a warning on the run's page when an + # upload only passed on a retry: the files are compared afterwards either + # way, but it should not go unnoticed again. - name: Upload this version run: | - set -euo pipefail - rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \ - --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line 2>&1 | tee upload.log - if grep -q "Failed to copy" upload.log; then - echo "::warning::rclone needed a retry: $(grep -c "Failed to copy" upload.log) uploads failed on an earlier attempt" - fi + ./test/vm-image/nightly-rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \ + --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line # Fetched the way a user fetches them. It proves what is served, and it # is the first request the CDN sees for each file. @@ -318,7 +314,7 @@ jobs: - name: Point latest.json at it run: | - rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket + ./test/vm-image/nightly-rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version) [ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; } echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY" diff --git a/test/unit/nightly-test.sh b/test/unit/nightly-test.sh index f17bd5d..ea4ff23 100644 --- a/test/unit/nightly-test.sh +++ b/test/unit/nightly-test.sh @@ -187,6 +187,33 @@ else echo "nightly-fetch tests skipped: no zip or unzip here" fi +# -------------------------------------------------------------- nightly-rclone +wrapped="$ROOT/test/vm-image/nightly-rclone" +bash -n "$wrapped" +mkdir -p "$tmp/rc" +rclone_says() { # exit status, output lines...: a stand-in rclone + local status=$1; shift + { echo '#!/bin/bash'; printf 'echo %q >&2\n' "$@"; echo "exit $status"; } >"$tmp/rc/rclone" + chmod +x "$tmp/rc/rclone" +} +rclone_says 0 "Transferred: 3 / 3, 100%" +out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b) +[[ $out != *::warning::* ]] || fail "rclone: a clean upload was reported as a retry" +[[ $out == *"Transferred: 3 / 3"* ]] || fail "rclone: its output was not passed through" + +rclone_says 0 "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \ + "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \ + "<3>ERROR : unpacked/vmlinuz: Failed to copy: NotImplemented: Not Implemented" \ + "<3>ERROR : Attempt 2/3 failed with 2 errors" "<3>ERROR : Attempt 3/3 succeeded" +out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b) +[[ $out == *"::warning::rclone needed a retry for 2 file(s): a.qcow2 unpacked/vmlinuz"* ]] || + fail "rclone: a file that failed twice is not counted once: $(grep warning <<<"$out")" + +rclone_says 3 "<3>ERROR : a.qcow2: Failed to copy: AccessDenied" +if PATH="$tmp/rc:$PATH" "$wrapped" copy a b >"$tmp/rc/out" 2>&1; then fail "rclone: a failed upload passed"; fi +equal "rclone: its exit status is kept" "$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b >/dev/null 2>&1; echo $?)" "3" +if grep -q '::warning::' "$tmp/rc/out"; then fail "rclone: a failed upload was reported as a retry"; fi + # ------------------------------------------------------------ the install test # The harness is a script that starts a VM when it is read; take the two # functions out of it. diff --git a/test/vm-image/nightly-rclone b/test/vm-image/nightly-rclone new file mode 100755 index 0000000..a9b9eb7 --- /dev/null +++ b/test/vm-image/nightly-rclone @@ -0,0 +1,19 @@ +#!/bin/bash +# Runs rclone for the publish job and says when it only got there on a retry. +# +# nightly-rclone +# +# rclone tries a failed transfer again by itself, and a run that passes on the +# second attempt looks like any other. This passes rclone's output and exit +# status through, and adds a warning for the run's page naming the files that +# failed on an earlier attempt. +set -uo pipefail +log=$(mktemp) +trap 'rm -f "$log"' EXIT +rclone "$@" 2>&1 | tee "$log" +status=${PIPESTATUS[0]} +files=$(sed -nE 's/^.*ERROR : (.*): Failed to copy: .*$/\1/p' "$log" | sort -u) +if [[ $status -eq 0 && -n $files ]]; then + echo "::warning::rclone needed a retry for $(wc -l <<<"$files") file(s): $(paste -sd' ' <<<"$files")" +fi +exit "$status" From 21a44ee4b494b847290c02793c7689281468d757 Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Fri, 9 Oct 2026 22:58:33 +0200 Subject: [PATCH 4/4] Pull the Arch image from GitHub's registry A unit test run stopped before it began: Docker Hub refused to hand out archlinux:latest, "You have reached your unauthenticated pull rate limit". GitHub's runners share addresses, so the limit is not ours to stay under. Arch publishes the same image as ghcr.io/archlinux/archlinux, which has no such limit from a runner. The three workflow containers and the ISO build's own container pull it from there. --- .github/workflows/install-test.yml | 2 +- .github/workflows/nightly-build.yml | 2 +- .github/workflows/unit-test.yml | 2 +- bin/omarchy-iso-make | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/install-test.yml b/.github/workflows/install-test.yml index 1e8470e..761ab7f 100644 --- a/.github/workflows/install-test.yml +++ b/.github/workflows/install-test.yml @@ -50,7 +50,7 @@ jobs: sudo chmod 0666 /dev/kvm docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \ -e OMARCHY_INTEGRATION_DISK_CACHE=none \ - archlinux:latest bash -euo pipefail -c ' + ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c ' # pacman 7 fails every sync on a kernel without Landlock, which # these runners have; builder/build-iso.sh does the same. if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 5bd0333..f718bec 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -132,7 +132,7 @@ jobs: sudo chmod 0666 /dev/kvm docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \ -e OMARCHY_INTEGRATION_DISK_CACHE=none \ - archlinux:latest bash -euo pipefail -c ' + ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c ' # pacman 7 fails every sync on a kernel without Landlock, which # these runners have; builder/build-iso.sh does the same. if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 137346f..e13df6c 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -33,7 +33,7 @@ jobs: run: | docker run -d --name unit --privileged --cgroupns=host \ -v /sys/fs/cgroup:/sys/fs/cgroup:rw --tmpfs /run --tmpfs /tmp:exec \ - -v "$PWD:/src:ro" archlinux:latest bash -c ' + -v "$PWD:/src:ro" ghcr.io/archlinux/archlinux:latest bash -c ' systemctl mask getty.target console-getty.service \ systemd-udev-trigger.service systemd-udevd.service \ systemd-vconsole-setup.service diff --git a/bin/omarchy-iso-make b/bin/omarchy-iso-make index 5dbbeff..0315c3f 100755 --- a/bin/omarchy-iso-make +++ b/bin/omarchy-iso-make @@ -166,7 +166,7 @@ if ! docker version &>/dev/null; then DOCKER=(sudo docker) fi -"${DOCKER[@]}" run "${DOCKER_ARGS[@]}" archlinux/archlinux:latest /$BUILD_SCRIPT +"${DOCKER[@]}" run "${DOCKER_ARGS[@]}" ghcr.io/archlinux/archlinux:latest /$BUILD_SCRIPT latest_iso=$(\ls -t "$BUILD_RELEASE_PATH"/*.iso | head -n1) iso_ref="${latest_iso%.*}-$OMARCHY_ISO_REF.iso"