Merge pull request #246 from omacom/ci/current-rclone
Nightly: a current rclone for the upload, and Ubuntu 26.04 runners
This commit is contained in:
8 files changed
+89
-16
No files matched your search
@@ -18,7 +18,7 @@ permissions:
|
||||
jobs:
|
||||
comment:
|
||||
if: github.event.workflow_run.event == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-26.04
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
# GitHub's runner: it is somebody else's code, and it has no business on
|
||||
# our account's machines. No cache volume either way: what one pull
|
||||
# request leaves in a cache the next run would trust.
|
||||
runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-24.04' || 'namespace-profile-linux-amd64-16x32' }}
|
||||
runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-26.04' || 'namespace-profile-linux-amd64-16x32' }}
|
||||
timeout-minutes: 75
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -43,14 +43,14 @@ jobs:
|
||||
run: NO_BOOT_OFFER=1 ./bin/omarchy-iso-make
|
||||
|
||||
# The harness runs in an Arch container: Arch's current QEMU and OVMF
|
||||
# (QEMU 11) instead of Ubuntu 24.04's QEMU 8.2, at the paths the
|
||||
# (QEMU 11) instead of the runner's older one, at the paths the
|
||||
# harness expects, the same as on an Omarchy workstation.
|
||||
- name: Install unattended, then boot the installed system
|
||||
run: |
|
||||
sudo chmod 0666 /dev/kvm
|
||||
docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
|
||||
-e OMARCHY_INTEGRATION_DISK_CACHE=none \
|
||||
archlinux:latest bash -euo pipefail -c '
|
||||
ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c '
|
||||
# pacman 7 fails every sync on a kernel without Landlock, which
|
||||
# these runners have; builder/build-iso.sh does the same.
|
||||
if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
# test/vm-image/nightly-plan decides, from what is fetched here.
|
||||
plan:
|
||||
name: Plan
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
channels: ${{ steps.plan.outputs.channels }}
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
sudo chmod 0666 /dev/kvm
|
||||
docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
|
||||
-e OMARCHY_INTEGRATION_DISK_CACHE=none \
|
||||
archlinux:latest bash -euo pipefail -c '
|
||||
ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c '
|
||||
# pacman 7 fails every sync on a kernel without Landlock, which
|
||||
# these runners have; builder/build-iso.sh does the same.
|
||||
if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
|
||||
@@ -229,7 +229,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include: ${{ fromJSON(needs.plan.outputs.channels) }}
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 45
|
||||
environment: nightly
|
||||
# actions: read is for finding and fetching the build job's artifact.
|
||||
@@ -258,21 +258,39 @@ jobs:
|
||||
- name: Fetch the release from the build job
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
# Not /tmp: on Ubuntu 26.04 it is held in memory, and the zip is
|
||||
# gigabytes.
|
||||
TMPDIR: ${{ runner.temp }}
|
||||
run: |
|
||||
sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null
|
||||
df -h "$TMPDIR" .
|
||||
command -v aria2c unzip >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null; }
|
||||
./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out
|
||||
|
||||
# A current rclone, by version and checksum. Ubuntu's is 1.60 from 2022:
|
||||
# with it R2 answered every first upload "501 Not Implemented", though
|
||||
# it had stored the file, and the publish only passed on rclone's retry.
|
||||
# 1.60 sends "x-amz-acl: private" with each upload; current rclone does
|
||||
# not send it to Cloudflare.
|
||||
- name: rclone
|
||||
env:
|
||||
RCLONE_VERSION: v1.75.2
|
||||
RCLONE_SHA256: efbfe852181f7191eb3c9043ed1ab49c9b2d0ba045c61c926a5da111c939ec5c
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ -n "$RCLONE_CONFIG_R2_ACCESS_KEY_ID" ] || { echo "::error::the nightly environment has no bucket key"; exit 1; }
|
||||
sudo apt-get update -qq && sudo apt-get install -y -qq rclone >/dev/null
|
||||
curl -fsSL --retry 3 -o rclone.deb "https://github.com/rclone/rclone/releases/download/$RCLONE_VERSION/rclone-$RCLONE_VERSION-linux-amd64.deb"
|
||||
echo "$RCLONE_SHA256 rclone.deb" | sha256sum -c -
|
||||
sudo dpkg -i rclone.deb >/dev/null
|
||||
rclone version | head -1
|
||||
|
||||
# --immutable: a name that exists with other bytes is an error, not an
|
||||
# overwrite.
|
||||
# nightly-rclone is rclone, with a warning on the run's page when an
|
||||
# upload only passed on a retry: the files are compared afterwards either
|
||||
# way, but it should not go unnoticed again.
|
||||
- name: Upload this version
|
||||
run: |
|
||||
rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
|
||||
./test/vm-image/nightly-rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
|
||||
--immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line
|
||||
|
||||
# Fetched the way a user fetches them. It proves what is served, and it
|
||||
@@ -296,7 +314,7 @@ jobs:
|
||||
|
||||
- name: Point latest.json at it
|
||||
run: |
|
||||
rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
|
||||
./test/vm-image/nightly-rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
|
||||
served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version)
|
||||
[ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; }
|
||||
echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY"
|
||||
@@ -15,7 +15,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-26.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
run: |
|
||||
docker run -d --name unit --privileged --cgroupns=host \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw --tmpfs /run --tmpfs /tmp:exec \
|
||||
-v "$PWD:/src:ro" archlinux:latest bash -c '
|
||||
-v "$PWD:/src:ro" ghcr.io/archlinux/archlinux:latest bash -c '
|
||||
systemctl mask getty.target console-getty.service \
|
||||
systemd-udev-trigger.service systemd-udevd.service \
|
||||
systemd-vconsole-setup.service
|
||||
|
||||
@@ -166,7 +166,7 @@ if ! docker version &>/dev/null; then
|
||||
DOCKER=(sudo docker)
|
||||
fi
|
||||
|
||||
"${DOCKER[@]}" run "${DOCKER_ARGS[@]}" archlinux/archlinux:latest /$BUILD_SCRIPT
|
||||
"${DOCKER[@]}" run "${DOCKER_ARGS[@]}" ghcr.io/archlinux/archlinux:latest /$BUILD_SCRIPT
|
||||
|
||||
latest_iso=$(\ls -t "$BUILD_RELEASE_PATH"/*.iso | head -n1)
|
||||
iso_ref="${latest_iso%.*}-$OMARCHY_ISO_REF.iso"
|
||||
|
||||
@@ -181,10 +181,39 @@ STUB
|
||||
refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3"
|
||||
refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4"
|
||||
[[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked"
|
||||
refused "an artifact larger than the free space" env FETCH_SIZE=9000000000000000 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f5"
|
||||
grep -q "MiB free" "$tmp/refused.err" || fail "fetch: too little room is not what was reported"
|
||||
else
|
||||
echo "nightly-fetch tests skipped: no zip or unzip here"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------------- nightly-rclone
|
||||
wrapped="$ROOT/test/vm-image/nightly-rclone"
|
||||
bash -n "$wrapped"
|
||||
mkdir -p "$tmp/rc"
|
||||
rclone_says() { # exit status, output lines...: a stand-in rclone
|
||||
local status=$1; shift
|
||||
{ echo '#!/bin/bash'; printf 'echo %q >&2\n' "$@"; echo "exit $status"; } >"$tmp/rc/rclone"
|
||||
chmod +x "$tmp/rc/rclone"
|
||||
}
|
||||
rclone_says 0 "Transferred: 3 / 3, 100%"
|
||||
out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b)
|
||||
[[ $out != *::warning::* ]] || fail "rclone: a clean upload was reported as a retry"
|
||||
[[ $out == *"Transferred: 3 / 3"* ]] || fail "rclone: its output was not passed through"
|
||||
|
||||
rclone_says 0 "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \
|
||||
"<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \
|
||||
"<3>ERROR : unpacked/vmlinuz: Failed to copy: NotImplemented: Not Implemented" \
|
||||
"<3>ERROR : Attempt 2/3 failed with 2 errors" "<3>ERROR : Attempt 3/3 succeeded"
|
||||
out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b)
|
||||
[[ $out == *"::warning::rclone needed a retry for 2 file(s): a.qcow2 unpacked/vmlinuz"* ]] ||
|
||||
fail "rclone: a file that failed twice is not counted once: $(grep warning <<<"$out")"
|
||||
|
||||
rclone_says 3 "<3>ERROR : a.qcow2: Failed to copy: AccessDenied"
|
||||
if PATH="$tmp/rc:$PATH" "$wrapped" copy a b >"$tmp/rc/out" 2>&1; then fail "rclone: a failed upload passed"; fi
|
||||
equal "rclone: its exit status is kept" "$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b >/dev/null 2>&1; echo $?)" "3"
|
||||
if grep -q '::warning::' "$tmp/rc/out"; then fail "rclone: a failed upload was reported as a retry"; fi
|
||||
|
||||
# ------------------------------------------------------------ the install test
|
||||
# The harness is a script that starts a VM when it is read; take the two
|
||||
# functions out of it.
|
||||
|
||||
@@ -8,11 +8,14 @@
|
||||
# release would outlast the job. An artifact is one zip at a signed address,
|
||||
# so this asks the API for that address and takes it over 16 connections.
|
||||
#
|
||||
# The zip goes to TMPDIR and is deleted once unpacked, so there has to be room
|
||||
# for the artifact twice: point TMPDIR at a disk, not at a /tmp held in memory.
|
||||
#
|
||||
# Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c
|
||||
# and unzip. A run started again keeps the artifacts of its earlier attempts
|
||||
# under the same name: the newest is the one fetched.
|
||||
set -euo pipefail
|
||||
[[ $# -eq 4 ]] || { sed -n '2,13p' "$0" >&2; exit 2; }
|
||||
[[ $# -eq 4 ]] || { sed -n '2,16p' "$0" >&2; exit 2; }
|
||||
repo=$1 run=$2 name=$3 out=$4
|
||||
fail() { echo "nightly-fetch: $*" >&2; exit 1; }
|
||||
|
||||
@@ -23,6 +26,11 @@ size=$(gh api "repos/$repo/actions/artifacts/$id" --jq .size_in_bytes)
|
||||
echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB"
|
||||
|
||||
zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip")
|
||||
mkdir -p "$out"
|
||||
for place in "$(dirname "$zip")" "$out"; do
|
||||
free=$(( $(df --output=avail -B1 "$place" | tail -n 1) ))
|
||||
(( free > size )) || fail "$place has $((free / 1048576)) MiB free, the artifact is $((size / 1048576)) MiB"
|
||||
done
|
||||
trap 'rm -f "$zip" "$zip.aria2"' EXIT
|
||||
started=$(date +%s)
|
||||
# The address is signed for a few minutes. A try that fails asks for a new one
|
||||
@@ -44,6 +52,5 @@ done
|
||||
seconds=$(( $(date +%s) - started ))
|
||||
echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s"
|
||||
|
||||
mkdir -p "$out"
|
||||
unzip -q "$zip" -d "$out"
|
||||
echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Runs rclone for the publish job and says when it only got there on a retry.
|
||||
#
|
||||
# nightly-rclone <rclone arguments...>
|
||||
#
|
||||
# rclone tries a failed transfer again by itself, and a run that passes on the
|
||||
# second attempt looks like any other. This passes rclone's output and exit
|
||||
# status through, and adds a warning for the run's page naming the files that
|
||||
# failed on an earlier attempt.
|
||||
set -uo pipefail
|
||||
log=$(mktemp)
|
||||
trap 'rm -f "$log"' EXIT
|
||||
rclone "$@" 2>&1 | tee "$log"
|
||||
status=${PIPESTATUS[0]}
|
||||
files=$(sed -nE 's/^.*ERROR : (.*): Failed to copy: .*$/\1/p' "$log" | sort -u)
|
||||
if [[ $status -eq 0 && -n $files ]]; then
|
||||
echo "::warning::rclone needed a retry for $(wc -l <<<"$files") file(s): $(paste -sd' ' <<<"$files")"
|
||||
fi
|
||||
exit "$status"
|
||||
Reference in new issue
Block a user