diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index c8437e5..a54a7ae 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -232,6 +232,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 45 environment: nightly + # actions: read is for finding and fetching the build job's artifact. + permissions: + contents: read + actions: read env: CHANNEL: ${{ matrix.channel }} VERSION: ${{ matrix.version }} @@ -243,10 +247,20 @@ jobs: RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} BUCKET: omarchy-nightly steps: - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - name: nightly-${{ matrix.channel }}-${{ github.run_id }} - path: out + sparse-checkout: test/vm-image + persist-credentials: false + + # Not the download action: GitHub's storage limits each connection to a + # few megabytes a second, and the action took 10 minutes over 4.6 GB. + # nightly-fetch takes the same artifact over 16 connections. + - name: Fetch the release from the build job + env: + GH_TOKEN: ${{ github.token }} + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null + ./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out - name: rclone run: | diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 84171f9..823b6bd 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -50,7 +50,7 @@ jobs: # mirror that drops one file must not fail the tests. sed -i "1i Server = https://mirror.omarchy.org/\$repo/os/\$arch" /etc/pacman.d/mirrorlist for attempt in 1 2 3; do - pacman -Syu --noconfirm --needed python parted diffutils jq >/dev/null && break + pacman -Syu --noconfirm --needed python parted diffutils jq zip unzip >/dev/null && break [ "$attempt" -lt 3 ] || { echo "::error::the test container could not install its tools"; exit 1; } sleep 15 done diff --git a/test/unit/nightly-test.sh b/test/unit/nightly-test.sh index 9706696..cf7624d 100644 --- a/test/unit/nightly-test.sh +++ b/test/unit/nightly-test.sh @@ -135,6 +135,56 @@ refused "a released ISO under another version's name" "$layout" stable 4.0.5 sta sealed "$tmp/sealed-z" refused "a channel that does not exist" "$layout" rc 4.0.4 rc/4.0.4 "$tmp/release/omarchy-4.0.4.iso" "$tmp/sealed-z" "$tmp/stable-packages" "$tmp/out5" +# --------------------------------------------------------------- nightly-fetch +# The network is played by three stand-ins: gh lists the artifacts, curl gives +# the signed address, aria2c "downloads" a zip made here. +fetch="$ROOT/test/vm-image/nightly-fetch" +bash -n "$fetch" +if command -v zip >/dev/null && command -v unzip >/dev/null; then + stubs=$tmp/stubs; mkdir -p "$stubs" "$tmp/payload/edge/1" + echo "an image" >"$tmp/payload/edge/1/omarchy-edge-1.qcow2" + echo '{"version": "1"}' >"$tmp/payload/edge/latest.json" + (cd "$tmp/payload" && zip -qr "$tmp/artifact.zip" .) + zip_size=$(stat -c %s "$tmp/artifact.zip") + cat >"$stubs/gh" <"$stubs/curl" <<'STUB' +#!/bin/bash +echo "${@: -1}" >>"$FETCH_LOG"; printf 'https://storage.example/signed' +STUB + cat >"$stubs/aria2c" <>"\$FETCH_LOG" +if [[ -n \${FETCH_FAIL_ONCE:-} && ! -e $tmp/failed ]]; then touch $tmp/failed; exit 1; fi +for arg in "\$@"; do case \$arg in --dir=*) dir=\${arg#--dir=} ;; --out=*) out=\${arg#--out=} ;; esac; done +cp "$tmp/artifact.zip" "\$dir/\$out" +STUB + chmod +x "$stubs"/* + fetched() { PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$@"; } + + : >"$tmp/fetch.log" + fetched "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/fetched" >/dev/null + equal "fetch: the files of the artifact" "$(cd "$tmp/fetched" && find . -type f | sort | paste -sd' ')" "./edge/1/omarchy-edge-1.qcow2 ./edge/latest.json" + grep -q "/artifacts/12/zip" "$tmp/fetch.log" || fail "fetch: of two artifacts with one name, the newer was not the one fetched" + equal "fetch: nothing left behind" "$(ls "$tmp" | grep -c 'nightly-fetch\.')" "0" + + : >"$tmp/fetch.log"; rm -f "$tmp/failed" + FETCH_FAIL_ONCE=1 fetched "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/fetched-again" >/dev/null 2>&1 + equal "fetch: a failed download is tried again with a new address" "$(grep -c zip "$tmp/fetch.log")" "2" + + refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3" + refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4" + [[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked" +else + echo "nightly-fetch tests skipped: no zip or unzip here" +fi + # ------------------------------------------------------------ the install test # The harness is a script that starts a VM when it is read; take the two # functions out of it. diff --git a/test/vm-image/nightly-fetch b/test/vm-image/nightly-fetch new file mode 100755 index 0000000..723c4a5 --- /dev/null +++ b/test/vm-image/nightly-fetch @@ -0,0 +1,49 @@ +#!/bin/bash +# Fetches a run's artifact for the publish job, and fast. +# +# nightly-fetch +# +# The download action takes a multi-gigabyte artifact over a few connections, +# and GitHub's storage limits each one to a few megabytes a second: an 11 GB +# release would outlast the job. An artifact is one zip at a signed address, +# so this asks the API for that address and takes it over 16 connections. +# +# Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c +# and unzip. A run started again keeps the artifacts of its earlier attempts +# under the same name: the newest is the one fetched. +set -euo pipefail +[[ $# -eq 4 ]] || { sed -n '2,13p' "$0" >&2; exit 2; } +repo=$1 run=$2 name=$3 out=$4 +fail() { echo "nightly-fetch: $*" >&2; exit 1; } + +id=$(gh api "repos/$repo/actions/runs/$run/artifacts" --paginate \ + --jq ".artifacts[] | select(.name == \"$name\" and .expired == false) | .id" | sort -n | tail -n 1) +[[ -n $id ]] || fail "run $run has no artifact named $name" +size=$(gh api "repos/$repo/actions/artifacts/$id" --jq .size_in_bytes) +echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB" + +zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip") +trap 'rm -f "$zip" "$zip.aria2"' EXIT +started=$(date +%s) +# The address is signed for a few minutes. A try that fails asks for a new one +# and carries on from what is already there. +for attempt in 1 2 3; do + url=$(curl -fsS -o /dev/null -w '%{redirect_url}' \ + -H "Authorization: Bearer $(gh auth token)" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$repo/actions/artifacts/$id/zip") + [[ $url == https://* ]] || fail "the API gave no download address for artifact $id" + if aria2c --quiet --continue --max-connection-per-server=16 --split=16 --min-split-size=16M \ + --file-allocation=none --dir="$(dirname "$zip")" --out="$(basename "$zip")" "$url"; then + break + fi + (( attempt < 3 )) || fail "the download failed three times" + echo "nightly-fetch: attempt $attempt failed, trying again" >&2 + sleep 10 +done +[[ $(stat -c %s "$zip") == "$size" ]] || fail "downloaded $(stat -c %s "$zip") bytes, the artifact is $size" +seconds=$(( $(date +%s) - started )) +echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s" + +mkdir -p "$out" +unzip -q "$zip" -d "$out" +echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out"