Merge pull request #245 from omacom/ci/bump-actions-dependabot

CI: latest actions, Dependabot weekly, and a fast fetch for the publish job

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# Every action is pinned to a commit, so nothing updates by itself. Once a
+# week Dependabot proposes the newer releases in one pull request, moving each
+# commit and the version in its comment together.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      actions:
+        patterns: ["*"]

.github/workflows/install-test.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-24.04' || 'namespace-profile-linux-amd64-16x32' }}
     timeout-minutes: 75
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       # Fail early and loudly if the runner ever has less disk than this needs.
       - name: Check disk space
@@ -37,7 +37,7 @@ jobs:
           avail_gb=$(df -BG --output=avail / | tail -1 | tr -dc 0-9)
           [ "$avail_gb" -ge 40 ] || { echo "::error::only ${avail_gb} GB free, the ISO build and install test need ~40 GB"; exit 1; }
 
-      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       - name: Build the ISO
         run: NO_BOOT_OFFER=1 ./bin/omarchy-iso-make
@@ -118,7 +118,7 @@ jobs:
 
       - name: Upload test artifacts
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: install-test-${{ github.run_id }}-${{ github.run_attempt }}
           path: |

.github/workflows/nightly-build.yml

@@ -55,7 +55,7 @@ jobs:
     outputs:
       channels: ${{ steps.plan.outputs.channels }}
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - id: plan
         env:
@@ -82,7 +82,7 @@ jobs:
       VERSION: ${{ matrix.version }}
       RELEASE: ${{ matrix.release }}
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Runner
         run: |
@@ -98,7 +98,7 @@ jobs:
           avail_gb=$(df -BG --output=avail / | tail -1 | tr -dc 0-9)
           [ "$avail_gb" -ge 50 ] || { echo "::error::only ${avail_gb} GB free, the build, the install test and the image need ~50 GB"; exit 1; }
 
-      - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
         if: matrix.channel == 'edge'
 
       # No package cache, on purpose: every package the ISO is made of is
@@ -190,7 +190,7 @@ jobs:
           } >>"$GITHUB_STEP_SUMMARY"
 
       - name: Keep the release for the publish job
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: nightly-${{ matrix.channel }}-${{ github.run_id }}
           path: out/
@@ -200,7 +200,7 @@ jobs:
 
       - name: Keep the install test's logs
         if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: nightly-install-test-${{ matrix.channel }}-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
@@ -232,6 +232,10 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 45
     environment: nightly
+    # actions: read is for finding and fetching the build job's artifact.
+    permissions:
+      contents: read
+      actions: read
     env:
       CHANNEL: ${{ matrix.channel }}
       VERSION: ${{ matrix.version }}
@@ -243,10 +247,20 @@ jobs:
       RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
       BUCKET: omarchy-nightly
     steps:
-      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          name: nightly-${{ matrix.channel }}-${{ github.run_id }}
+          sparse-checkout: test/vm-image
-          path: out
+          persist-credentials: false
+
+      # Not the download action: GitHub's storage limits each connection to a
+      # few megabytes a second, and the action took 10 minutes over 4.6 GB.
+      # nightly-fetch takes the same artifact over 16 connections.
+      - name: Fetch the release from the build job
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null
+          ./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out
 
       - name: rclone
         run: |

.github/workflows/unit-test.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       # The tests run where the code they cover runs: on Arch, the same as
       # the live ISO and an Omarchy workstation. The container boots systemd
@@ -50,7 +50,7 @@ jobs:
             # mirror that drops one file must not fail the tests.
             sed -i "1i Server = https://mirror.omarchy.org/\$repo/os/\$arch" /etc/pacman.d/mirrorlist
             for attempt in 1 2 3; do
-              pacman -Syu --noconfirm --needed python parted diffutils jq >/dev/null && break
+              pacman -Syu --noconfirm --needed python parted diffutils jq zip unzip >/dev/null && break
               [ "$attempt" -lt 3 ] || { echo "::error::the test container could not install its tools"; exit 1; }
               sleep 15
             done

test/unit/nightly-test.sh

@@ -135,6 +135,56 @@ refused "a released ISO under another version's name" "$layout" stable 4.0.5 sta
 sealed "$tmp/sealed-z"
 refused "a channel that does not exist" "$layout" rc 4.0.4 rc/4.0.4 "$tmp/release/omarchy-4.0.4.iso" "$tmp/sealed-z" "$tmp/stable-packages" "$tmp/out5"
 
+# --------------------------------------------------------------- nightly-fetch
+# The network is played by three stand-ins: gh lists the artifacts, curl gives
+# the signed address, aria2c "downloads" a zip made here.
+fetch="$ROOT/test/vm-image/nightly-fetch"
+bash -n "$fetch"
+if command -v zip >/dev/null && command -v unzip >/dev/null; then
+  stubs=$tmp/stubs; mkdir -p "$stubs" "$tmp/payload/edge/1"
+  echo "an image" >"$tmp/payload/edge/1/omarchy-edge-1.qcow2"
+  echo '{"version": "1"}' >"$tmp/payload/edge/latest.json"
+  (cd "$tmp/payload" && zip -qr "$tmp/artifact.zip" .)
+  zip_size=$(stat -c %s "$tmp/artifact.zip")
+  cat >"$stubs/gh" <<STUB
+#!/bin/bash
+case "\$*" in
+  "auth token") echo token ;;
+  *"/runs/7/artifacts"*) printf '%s\\n' \${FETCH_IDS:-11 12} ;;
+  *"/artifacts/"*) echo \${FETCH_SIZE:-$zip_size} ;;
+esac
+STUB
+  cat >"$stubs/curl" <<'STUB'
+#!/bin/bash
+echo "${@: -1}" >>"$FETCH_LOG"; printf 'https://storage.example/signed'
+STUB
+  cat >"$stubs/aria2c" <<STUB
+#!/bin/bash
+echo aria2c >>"\$FETCH_LOG"
+if [[ -n \${FETCH_FAIL_ONCE:-} && ! -e $tmp/failed ]]; then touch $tmp/failed; exit 1; fi
+for arg in "\$@"; do case \$arg in --dir=*) dir=\${arg#--dir=} ;; --out=*) out=\${arg#--out=} ;; esac; done
+cp "$tmp/artifact.zip" "\$dir/\$out"
+STUB
+  chmod +x "$stubs"/*
+  fetched() { PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$@"; }
+
+  : >"$tmp/fetch.log"
+  fetched "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/fetched" >/dev/null
+  equal "fetch: the files of the artifact" "$(cd "$tmp/fetched" && find . -type f | sort | paste -sd' ')" "./edge/1/omarchy-edge-1.qcow2 ./edge/latest.json"
+  grep -q "/artifacts/12/zip" "$tmp/fetch.log" || fail "fetch: of two artifacts with one name, the newer was not the one fetched"
+  equal "fetch: nothing left behind" "$(ls "$tmp" | grep -c 'nightly-fetch\.')" "0"
+
+  : >"$tmp/fetch.log"; rm -f "$tmp/failed"
+  FETCH_FAIL_ONCE=1 fetched "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/fetched-again" >/dev/null 2>&1
+  equal "fetch: a failed download is tried again with a new address" "$(grep -c zip "$tmp/fetch.log")" "2"
+
+  refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3"
+  refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4"
+  [[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked"
+else
+  echo "nightly-fetch tests skipped: no zip or unzip here"
+fi
+
 # ------------------------------------------------------------ the install test
 # The harness is a script that starts a VM when it is read; take the two
 # functions out of it.

test/vm-image/nightly-fetch

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Fetches a run's artifact for the publish job, and fast.
+#
+#   nightly-fetch <owner/repo> <run_id> <artifact_name> <out_dir>
+#
+# The download action takes a multi-gigabyte artifact over a few connections,
+# and GitHub's storage limits each one to a few megabytes a second: an 11 GB
+# release would outlast the job. An artifact is one zip at a signed address,
+# so this asks the API for that address and takes it over 16 connections.
+#
+# Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c
+# and unzip. A run started again keeps the artifacts of its earlier attempts
+# under the same name: the newest is the one fetched.
+set -euo pipefail
+[[ $# -eq 4 ]] || { sed -n '2,13p' "$0" >&2; exit 2; }
+repo=$1 run=$2 name=$3 out=$4
+fail() { echo "nightly-fetch: $*" >&2; exit 1; }
+
+id=$(gh api "repos/$repo/actions/runs/$run/artifacts" --paginate \
+  --jq ".artifacts[] | select(.name == \"$name\" and .expired == false) | .id" | sort -n | tail -n 1)
+[[ -n $id ]] || fail "run $run has no artifact named $name"
+size=$(gh api "repos/$repo/actions/artifacts/$id" --jq .size_in_bytes)
+echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB"
+
+zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip")
+trap 'rm -f "$zip" "$zip.aria2"' EXIT
+started=$(date +%s)
+# The address is signed for a few minutes. A try that fails asks for a new one
+# and carries on from what is already there.
+for attempt in 1 2 3; do
+  url=$(curl -fsS -o /dev/null -w '%{redirect_url}' \
+    -H "Authorization: Bearer $(gh auth token)" -H "Accept: application/vnd.github+json" \
+    "https://api.github.com/repos/$repo/actions/artifacts/$id/zip")
+  [[ $url == https://* ]] || fail "the API gave no download address for artifact $id"
+  if aria2c --quiet --continue --max-connection-per-server=16 --split=16 --min-split-size=16M \
+    --file-allocation=none --dir="$(dirname "$zip")" --out="$(basename "$zip")" "$url"; then
+    break
+  fi
+  (( attempt < 3 )) || fail "the download failed three times"
+  echo "nightly-fetch: attempt $attempt failed, trying again" >&2
+  sleep 10
+done
+[[ $(stat -c %s "$zip") == "$size" ]] || fail "downloaded $(stat -c %s "$zip") bytes, the artifact is $size"
+seconds=$(( $(date +%s) - started ))
+echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s"
+
+mkdir -p "$out"
+unzip -q "$zip" -d "$out"
+echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out"