Nightly: fetch the release over 16 connections
The first publish spent 9 min 42 s in the download action for the 4.6 GB stable release; the 11.5 GB edge release would not have fitted in the job's 45 minutes. GitHub's artifact storage limits each connection to a few megabytes a second, and the action uses few. An artifact is one zip at a signed address. test/vm-image/nightly-fetch asks the API for that address and takes it with aria2c over 16 connections, checks the size and unpacks it. The same 4.6 GB artifact came down in 123 s that way. The publish job gets actions: read for it, and checks out test/vm-image for the script. test/unit/nightly-test.sh plays the network with stand-ins: the newest artifact of a name is the one fetched, a failed download is tried again with a new address, a wrong size is refused and not unpacked.
This commit is contained in:
Emir Beganovic committed
1 parent
bb1ad28887
commit
b01a396e62
4 files changed
+117
-4
No files matched your search
@@ -232,6 +232,10 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
environment: nightly
|
||||
# actions: read is for finding and fetching the build job's artifact.
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
CHANNEL: ${{ matrix.channel }}
|
||||
VERSION: ${{ matrix.version }}
|
||||
@@ -243,10 +247,20 @@ jobs:
|
||||
RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
BUCKET: omarchy-nightly
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
name: nightly-${{ matrix.channel }}-${{ github.run_id }}
|
||||
path: out
|
||||
sparse-checkout: test/vm-image
|
||||
persist-credentials: false
|
||||
|
||||
# Not the download action: GitHub's storage limits each connection to a
|
||||
# few megabytes a second, and the action took 10 minutes over 4.6 GB.
|
||||
# nightly-fetch takes the same artifact over 16 connections.
|
||||
- name: Fetch the release from the build job
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null
|
||||
./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out
|
||||
|
||||
- name: rclone
|
||||
run: |
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
# mirror that drops one file must not fail the tests.
|
||||
sed -i "1i Server = https://mirror.omarchy.org/\$repo/os/\$arch" /etc/pacman.d/mirrorlist
|
||||
for attempt in 1 2 3; do
|
||||
pacman -Syu --noconfirm --needed python parted diffutils jq >/dev/null && break
|
||||
pacman -Syu --noconfirm --needed python parted diffutils jq zip unzip >/dev/null && break
|
||||
[ "$attempt" -lt 3 ] || { echo "::error::the test container could not install its tools"; exit 1; }
|
||||
sleep 15
|
||||
done
|
||||
|
||||
@@ -135,6 +135,56 @@ refused "a released ISO under another version's name" "$layout" stable 4.0.5 sta
|
||||
sealed "$tmp/sealed-z"
|
||||
refused "a channel that does not exist" "$layout" rc 4.0.4 rc/4.0.4 "$tmp/release/omarchy-4.0.4.iso" "$tmp/sealed-z" "$tmp/stable-packages" "$tmp/out5"
|
||||
|
||||
# --------------------------------------------------------------- nightly-fetch
|
||||
# The network is played by three stand-ins: gh lists the artifacts, curl gives
|
||||
# the signed address, aria2c "downloads" a zip made here.
|
||||
fetch="$ROOT/test/vm-image/nightly-fetch"
|
||||
bash -n "$fetch"
|
||||
if command -v zip >/dev/null && command -v unzip >/dev/null; then
|
||||
stubs=$tmp/stubs; mkdir -p "$stubs" "$tmp/payload/edge/1"
|
||||
echo "an image" >"$tmp/payload/edge/1/omarchy-edge-1.qcow2"
|
||||
echo '{"version": "1"}' >"$tmp/payload/edge/latest.json"
|
||||
(cd "$tmp/payload" && zip -qr "$tmp/artifact.zip" .)
|
||||
zip_size=$(stat -c %s "$tmp/artifact.zip")
|
||||
cat >"$stubs/gh" <<STUB
|
||||
#!/bin/bash
|
||||
case "\$*" in
|
||||
"auth token") echo token ;;
|
||||
*"/runs/7/artifacts"*) printf '%s\\n' \${FETCH_IDS:-11 12} ;;
|
||||
*"/artifacts/"*) echo \${FETCH_SIZE:-$zip_size} ;;
|
||||
esac
|
||||
STUB
|
||||
cat >"$stubs/curl" <<'STUB'
|
||||
#!/bin/bash
|
||||
echo "${@: -1}" >>"$FETCH_LOG"; printf 'https://storage.example/signed'
|
||||
STUB
|
||||
cat >"$stubs/aria2c" <<STUB
|
||||
#!/bin/bash
|
||||
echo aria2c >>"\$FETCH_LOG"
|
||||
if [[ -n \${FETCH_FAIL_ONCE:-} && ! -e $tmp/failed ]]; then touch $tmp/failed; exit 1; fi
|
||||
for arg in "\$@"; do case \$arg in --dir=*) dir=\${arg#--dir=} ;; --out=*) out=\${arg#--out=} ;; esac; done
|
||||
cp "$tmp/artifact.zip" "\$dir/\$out"
|
||||
STUB
|
||||
chmod +x "$stubs"/*
|
||||
fetched() { PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$@"; }
|
||||
|
||||
: >"$tmp/fetch.log"
|
||||
fetched "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/fetched" >/dev/null
|
||||
equal "fetch: the files of the artifact" "$(cd "$tmp/fetched" && find . -type f | sort | paste -sd' ')" "./edge/1/omarchy-edge-1.qcow2 ./edge/latest.json"
|
||||
grep -q "/artifacts/12/zip" "$tmp/fetch.log" || fail "fetch: of two artifacts with one name, the newer was not the one fetched"
|
||||
equal "fetch: nothing left behind" "$(ls "$tmp" | grep -c 'nightly-fetch\.')" "0"
|
||||
|
||||
: >"$tmp/fetch.log"; rm -f "$tmp/failed"
|
||||
FETCH_FAIL_ONCE=1 fetched "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/fetched-again" >/dev/null 2>&1
|
||||
equal "fetch: a failed download is tried again with a new address" "$(grep -c zip "$tmp/fetch.log")" "2"
|
||||
|
||||
refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3"
|
||||
refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4"
|
||||
[[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked"
|
||||
else
|
||||
echo "nightly-fetch tests skipped: no zip or unzip here"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------ the install test
|
||||
# The harness is a script that starts a VM when it is read; take the two
|
||||
# functions out of it.
|
||||
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
# Fetches a run's artifact for the publish job, and fast.
|
||||
#
|
||||
# nightly-fetch <owner/repo> <run_id> <artifact_name> <out_dir>
|
||||
#
|
||||
# The download action takes a multi-gigabyte artifact over a few connections,
|
||||
# and GitHub's storage limits each one to a few megabytes a second: an 11 GB
|
||||
# release would outlast the job. An artifact is one zip at a signed address,
|
||||
# so this asks the API for that address and takes it over 16 connections.
|
||||
#
|
||||
# Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c
|
||||
# and unzip. A run started again keeps the artifacts of its earlier attempts
|
||||
# under the same name: the newest is the one fetched.
|
||||
set -euo pipefail
|
||||
[[ $# -eq 4 ]] || { sed -n '2,13p' "$0" >&2; exit 2; }
|
||||
repo=$1 run=$2 name=$3 out=$4
|
||||
fail() { echo "nightly-fetch: $*" >&2; exit 1; }
|
||||
|
||||
id=$(gh api "repos/$repo/actions/runs/$run/artifacts" --paginate \
|
||||
--jq ".artifacts[] | select(.name == \"$name\" and .expired == false) | .id" | sort -n | tail -n 1)
|
||||
[[ -n $id ]] || fail "run $run has no artifact named $name"
|
||||
size=$(gh api "repos/$repo/actions/artifacts/$id" --jq .size_in_bytes)
|
||||
echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB"
|
||||
|
||||
zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip")
|
||||
trap 'rm -f "$zip" "$zip.aria2"' EXIT
|
||||
started=$(date +%s)
|
||||
# The address is signed for a few minutes. A try that fails asks for a new one
|
||||
# and carries on from what is already there.
|
||||
for attempt in 1 2 3; do
|
||||
url=$(curl -fsS -o /dev/null -w '%{redirect_url}' \
|
||||
-H "Authorization: Bearer $(gh auth token)" -H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/$repo/actions/artifacts/$id/zip")
|
||||
[[ $url == https://* ]] || fail "the API gave no download address for artifact $id"
|
||||
if aria2c --quiet --continue --max-connection-per-server=16 --split=16 --min-split-size=16M \
|
||||
--file-allocation=none --dir="$(dirname "$zip")" --out="$(basename "$zip")" "$url"; then
|
||||
break
|
||||
fi
|
||||
(( attempt < 3 )) || fail "the download failed three times"
|
||||
echo "nightly-fetch: attempt $attempt failed, trying again" >&2
|
||||
sleep 10
|
||||
done
|
||||
[[ $(stat -c %s "$zip") == "$size" ]] || fail "downloaded $(stat -c %s "$zip") bytes, the artifact is $size"
|
||||
seconds=$(( $(date +%s) - started ))
|
||||
echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s"
|
||||
|
||||
mkdir -p "$out"
|
||||
unzip -q "$zip" -d "$out"
|
||||
echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out"
|
||||
Reference in new issue
Block a user