Merge pull request #246 from omacom/ci/current-rclone
Nightly: a current rclone for the upload, and Ubuntu 26.04 runners
This commit is contained in:
8 files changed
+89
-16
No files matched your search
@@ -18,7 +18,7 @@ permissions:
|
|||||||
jobs:
|
jobs:
|
||||||
comment:
|
comment:
|
||||||
if: github.event.workflow_run.event == 'pull_request'
|
if: github.event.workflow_run.event == 'pull_request'
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-26.04
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ github.token }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
REPO: ${{ github.repository }}
|
REPO: ${{ github.repository }}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ jobs:
|
|||||||
# GitHub's runner: it is somebody else's code, and it has no business on
|
# GitHub's runner: it is somebody else's code, and it has no business on
|
||||||
# our account's machines. No cache volume either way: what one pull
|
# our account's machines. No cache volume either way: what one pull
|
||||||
# request leaves in a cache the next run would trust.
|
# request leaves in a cache the next run would trust.
|
||||||
runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-24.04' || 'namespace-profile-linux-amd64-16x32' }}
|
runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-26.04' || 'namespace-profile-linux-amd64-16x32' }}
|
||||||
timeout-minutes: 75
|
timeout-minutes: 75
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
@@ -43,14 +43,14 @@ jobs:
|
|||||||
run: NO_BOOT_OFFER=1 ./bin/omarchy-iso-make
|
run: NO_BOOT_OFFER=1 ./bin/omarchy-iso-make
|
||||||
|
|
||||||
# The harness runs in an Arch container: Arch's current QEMU and OVMF
|
# The harness runs in an Arch container: Arch's current QEMU and OVMF
|
||||||
# (QEMU 11) instead of Ubuntu 24.04's QEMU 8.2, at the paths the
|
# (QEMU 11) instead of the runner's older one, at the paths the
|
||||||
# harness expects, the same as on an Omarchy workstation.
|
# harness expects, the same as on an Omarchy workstation.
|
||||||
- name: Install unattended, then boot the installed system
|
- name: Install unattended, then boot the installed system
|
||||||
run: |
|
run: |
|
||||||
sudo chmod 0666 /dev/kvm
|
sudo chmod 0666 /dev/kvm
|
||||||
docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
|
docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
|
||||||
-e OMARCHY_INTEGRATION_DISK_CACHE=none \
|
-e OMARCHY_INTEGRATION_DISK_CACHE=none \
|
||||||
archlinux:latest bash -euo pipefail -c '
|
ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c '
|
||||||
# pacman 7 fails every sync on a kernel without Landlock, which
|
# pacman 7 fails every sync on a kernel without Landlock, which
|
||||||
# these runners have; builder/build-iso.sh does the same.
|
# these runners have; builder/build-iso.sh does the same.
|
||||||
if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
|
if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ jobs:
|
|||||||
# test/vm-image/nightly-plan decides, from what is fetched here.
|
# test/vm-image/nightly-plan decides, from what is fetched here.
|
||||||
plan:
|
plan:
|
||||||
name: Plan
|
name: Plan
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-26.04
|
||||||
timeout-minutes: 5
|
timeout-minutes: 5
|
||||||
outputs:
|
outputs:
|
||||||
channels: ${{ steps.plan.outputs.channels }}
|
channels: ${{ steps.plan.outputs.channels }}
|
||||||
@@ -132,7 +132,7 @@ jobs:
|
|||||||
sudo chmod 0666 /dev/kvm
|
sudo chmod 0666 /dev/kvm
|
||||||
docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
|
docker run --rm --device /dev/kvm -v "$PWD:/w" -w /w \
|
||||||
-e OMARCHY_INTEGRATION_DISK_CACHE=none \
|
-e OMARCHY_INTEGRATION_DISK_CACHE=none \
|
||||||
archlinux:latest bash -euo pipefail -c '
|
ghcr.io/archlinux/archlinux:latest bash -euo pipefail -c '
|
||||||
# pacman 7 fails every sync on a kernel without Landlock, which
|
# pacman 7 fails every sync on a kernel without Landlock, which
|
||||||
# these runners have; builder/build-iso.sh does the same.
|
# these runners have; builder/build-iso.sh does the same.
|
||||||
if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
|
if ! grep -qw landlock /sys/kernel/security/lsm 2>/dev/null; then
|
||||||
@@ -229,7 +229,7 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
include: ${{ fromJSON(needs.plan.outputs.channels) }}
|
include: ${{ fromJSON(needs.plan.outputs.channels) }}
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-26.04
|
||||||
timeout-minutes: 45
|
timeout-minutes: 45
|
||||||
environment: nightly
|
environment: nightly
|
||||||
# actions: read is for finding and fetching the build job's artifact.
|
# actions: read is for finding and fetching the build job's artifact.
|
||||||
@@ -258,21 +258,39 @@ jobs:
|
|||||||
- name: Fetch the release from the build job
|
- name: Fetch the release from the build job
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ github.token }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
# Not /tmp: on Ubuntu 26.04 it is held in memory, and the zip is
|
||||||
|
# gigabytes.
|
||||||
|
TMPDIR: ${{ runner.temp }}
|
||||||
run: |
|
run: |
|
||||||
sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null
|
df -h "$TMPDIR" .
|
||||||
|
command -v aria2c unzip >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq aria2 unzip >/dev/null; }
|
||||||
./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out
|
./test/vm-image/nightly-fetch "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" "nightly-$CHANNEL-$GITHUB_RUN_ID" out
|
||||||
|
|
||||||
|
# A current rclone, by version and checksum. Ubuntu's is 1.60 from 2022:
|
||||||
|
# with it R2 answered every first upload "501 Not Implemented", though
|
||||||
|
# it had stored the file, and the publish only passed on rclone's retry.
|
||||||
|
# 1.60 sends "x-amz-acl: private" with each upload; current rclone does
|
||||||
|
# not send it to Cloudflare.
|
||||||
- name: rclone
|
- name: rclone
|
||||||
|
env:
|
||||||
|
RCLONE_VERSION: v1.75.2
|
||||||
|
RCLONE_SHA256: efbfe852181f7191eb3c9043ed1ab49c9b2d0ba045c61c926a5da111c939ec5c
|
||||||
run: |
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
[ -n "$RCLONE_CONFIG_R2_ACCESS_KEY_ID" ] || { echo "::error::the nightly environment has no bucket key"; exit 1; }
|
[ -n "$RCLONE_CONFIG_R2_ACCESS_KEY_ID" ] || { echo "::error::the nightly environment has no bucket key"; exit 1; }
|
||||||
sudo apt-get update -qq && sudo apt-get install -y -qq rclone >/dev/null
|
curl -fsSL --retry 3 -o rclone.deb "https://github.com/rclone/rclone/releases/download/$RCLONE_VERSION/rclone-$RCLONE_VERSION-linux-amd64.deb"
|
||||||
|
echo "$RCLONE_SHA256 rclone.deb" | sha256sum -c -
|
||||||
|
sudo dpkg -i rclone.deb >/dev/null
|
||||||
rclone version | head -1
|
rclone version | head -1
|
||||||
|
|
||||||
# --immutable: a name that exists with other bytes is an error, not an
|
# --immutable: a name that exists with other bytes is an error, not an
|
||||||
# overwrite.
|
# overwrite.
|
||||||
|
# nightly-rclone is rclone, with a warning on the run's page when an
|
||||||
|
# upload only passed on a retry: the files are compared afterwards either
|
||||||
|
# way, but it should not go unnoticed again.
|
||||||
- name: Upload this version
|
- name: Upload this version
|
||||||
run: |
|
run: |
|
||||||
rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
|
./test/vm-image/nightly-rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
|
||||||
--immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line
|
--immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line
|
||||||
|
|
||||||
# Fetched the way a user fetches them. It proves what is served, and it
|
# Fetched the way a user fetches them. It proves what is served, and it
|
||||||
@@ -296,7 +314,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Point latest.json at it
|
- name: Point latest.json at it
|
||||||
run: |
|
run: |
|
||||||
rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
|
./test/vm-image/nightly-rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
|
||||||
served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version)
|
served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version)
|
||||||
[ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; }
|
[ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; }
|
||||||
echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY"
|
echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY"
|
||||||
@@ -15,7 +15,7 @@ permissions:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
unit:
|
unit:
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-26.04
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
@@ -33,7 +33,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
docker run -d --name unit --privileged --cgroupns=host \
|
docker run -d --name unit --privileged --cgroupns=host \
|
||||||
-v /sys/fs/cgroup:/sys/fs/cgroup:rw --tmpfs /run --tmpfs /tmp:exec \
|
-v /sys/fs/cgroup:/sys/fs/cgroup:rw --tmpfs /run --tmpfs /tmp:exec \
|
||||||
-v "$PWD:/src:ro" archlinux:latest bash -c '
|
-v "$PWD:/src:ro" ghcr.io/archlinux/archlinux:latest bash -c '
|
||||||
systemctl mask getty.target console-getty.service \
|
systemctl mask getty.target console-getty.service \
|
||||||
systemd-udev-trigger.service systemd-udevd.service \
|
systemd-udev-trigger.service systemd-udevd.service \
|
||||||
systemd-vconsole-setup.service
|
systemd-vconsole-setup.service
|
||||||
|
|||||||
@@ -166,7 +166,7 @@ if ! docker version &>/dev/null; then
|
|||||||
DOCKER=(sudo docker)
|
DOCKER=(sudo docker)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
"${DOCKER[@]}" run "${DOCKER_ARGS[@]}" archlinux/archlinux:latest /$BUILD_SCRIPT
|
"${DOCKER[@]}" run "${DOCKER_ARGS[@]}" ghcr.io/archlinux/archlinux:latest /$BUILD_SCRIPT
|
||||||
|
|
||||||
latest_iso=$(\ls -t "$BUILD_RELEASE_PATH"/*.iso | head -n1)
|
latest_iso=$(\ls -t "$BUILD_RELEASE_PATH"/*.iso | head -n1)
|
||||||
iso_ref="${latest_iso%.*}-$OMARCHY_ISO_REF.iso"
|
iso_ref="${latest_iso%.*}-$OMARCHY_ISO_REF.iso"
|
||||||
|
|||||||
@@ -181,10 +181,39 @@ STUB
|
|||||||
refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3"
|
refused "an artifact that does not exist" env FETCH_IDS=" " PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f3"
|
||||||
refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4"
|
refused "a download of the wrong size" env FETCH_SIZE=1 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f4"
|
||||||
[[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked"
|
[[ ! -e $tmp/f4/edge ]] || fail "fetch: a download of the wrong size was unpacked"
|
||||||
|
refused "an artifact larger than the free space" env FETCH_SIZE=9000000000000000 PATH="$stubs:$PATH" TMPDIR="$tmp" FETCH_LOG="$tmp/fetch.log" "$fetch" omacom/omarchy-iso 7 nightly-edge-7 "$tmp/f5"
|
||||||
|
grep -q "MiB free" "$tmp/refused.err" || fail "fetch: too little room is not what was reported"
|
||||||
else
|
else
|
||||||
echo "nightly-fetch tests skipped: no zip or unzip here"
|
echo "nightly-fetch tests skipped: no zip or unzip here"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# -------------------------------------------------------------- nightly-rclone
|
||||||
|
wrapped="$ROOT/test/vm-image/nightly-rclone"
|
||||||
|
bash -n "$wrapped"
|
||||||
|
mkdir -p "$tmp/rc"
|
||||||
|
rclone_says() { # exit status, output lines...: a stand-in rclone
|
||||||
|
local status=$1; shift
|
||||||
|
{ echo '#!/bin/bash'; printf 'echo %q >&2\n' "$@"; echo "exit $status"; } >"$tmp/rc/rclone"
|
||||||
|
chmod +x "$tmp/rc/rclone"
|
||||||
|
}
|
||||||
|
rclone_says 0 "Transferred: 3 / 3, 100%"
|
||||||
|
out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b)
|
||||||
|
[[ $out != *::warning::* ]] || fail "rclone: a clean upload was reported as a retry"
|
||||||
|
[[ $out == *"Transferred: 3 / 3"* ]] || fail "rclone: its output was not passed through"
|
||||||
|
|
||||||
|
rclone_says 0 "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \
|
||||||
|
"<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \
|
||||||
|
"<3>ERROR : unpacked/vmlinuz: Failed to copy: NotImplemented: Not Implemented" \
|
||||||
|
"<3>ERROR : Attempt 2/3 failed with 2 errors" "<3>ERROR : Attempt 3/3 succeeded"
|
||||||
|
out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b)
|
||||||
|
[[ $out == *"::warning::rclone needed a retry for 2 file(s): a.qcow2 unpacked/vmlinuz"* ]] ||
|
||||||
|
fail "rclone: a file that failed twice is not counted once: $(grep warning <<<"$out")"
|
||||||
|
|
||||||
|
rclone_says 3 "<3>ERROR : a.qcow2: Failed to copy: AccessDenied"
|
||||||
|
if PATH="$tmp/rc:$PATH" "$wrapped" copy a b >"$tmp/rc/out" 2>&1; then fail "rclone: a failed upload passed"; fi
|
||||||
|
equal "rclone: its exit status is kept" "$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b >/dev/null 2>&1; echo $?)" "3"
|
||||||
|
if grep -q '::warning::' "$tmp/rc/out"; then fail "rclone: a failed upload was reported as a retry"; fi
|
||||||
|
|
||||||
# ------------------------------------------------------------ the install test
|
# ------------------------------------------------------------ the install test
|
||||||
# The harness is a script that starts a VM when it is read; take the two
|
# The harness is a script that starts a VM when it is read; take the two
|
||||||
# functions out of it.
|
# functions out of it.
|
||||||
|
|||||||
@@ -8,11 +8,14 @@
|
|||||||
# release would outlast the job. An artifact is one zip at a signed address,
|
# release would outlast the job. An artifact is one zip at a signed address,
|
||||||
# so this asks the API for that address and takes it over 16 connections.
|
# so this asks the API for that address and takes it over 16 connections.
|
||||||
#
|
#
|
||||||
|
# The zip goes to TMPDIR and is deleted once unpacked, so there has to be room
|
||||||
|
# for the artifact twice: point TMPDIR at a disk, not at a /tmp held in memory.
|
||||||
|
#
|
||||||
# Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c
|
# Needs gh (with GH_TOKEN allowed to read the run's artifacts), curl, aria2c
|
||||||
# and unzip. A run started again keeps the artifacts of its earlier attempts
|
# and unzip. A run started again keeps the artifacts of its earlier attempts
|
||||||
# under the same name: the newest is the one fetched.
|
# under the same name: the newest is the one fetched.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
[[ $# -eq 4 ]] || { sed -n '2,13p' "$0" >&2; exit 2; }
|
[[ $# -eq 4 ]] || { sed -n '2,16p' "$0" >&2; exit 2; }
|
||||||
repo=$1 run=$2 name=$3 out=$4
|
repo=$1 run=$2 name=$3 out=$4
|
||||||
fail() { echo "nightly-fetch: $*" >&2; exit 1; }
|
fail() { echo "nightly-fetch: $*" >&2; exit 1; }
|
||||||
|
|
||||||
@@ -23,6 +26,11 @@ size=$(gh api "repos/$repo/actions/artifacts/$id" --jq .size_in_bytes)
|
|||||||
echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB"
|
echo "nightly-fetch: artifact $id, $((size / 1048576)) MiB"
|
||||||
|
|
||||||
zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip")
|
zip=$(mktemp -u "${TMPDIR:-/tmp}/nightly-fetch.XXXXXX.zip")
|
||||||
|
mkdir -p "$out"
|
||||||
|
for place in "$(dirname "$zip")" "$out"; do
|
||||||
|
free=$(( $(df --output=avail -B1 "$place" | tail -n 1) ))
|
||||||
|
(( free > size )) || fail "$place has $((free / 1048576)) MiB free, the artifact is $((size / 1048576)) MiB"
|
||||||
|
done
|
||||||
trap 'rm -f "$zip" "$zip.aria2"' EXIT
|
trap 'rm -f "$zip" "$zip.aria2"' EXIT
|
||||||
started=$(date +%s)
|
started=$(date +%s)
|
||||||
# The address is signed for a few minutes. A try that fails asks for a new one
|
# The address is signed for a few minutes. A try that fails asks for a new one
|
||||||
@@ -44,6 +52,5 @@ done
|
|||||||
seconds=$(( $(date +%s) - started ))
|
seconds=$(( $(date +%s) - started ))
|
||||||
echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s"
|
echo "nightly-fetch: downloaded in $seconds s, $(( size / 1048576 / (seconds > 0 ? seconds : 1) )) MiB/s"
|
||||||
|
|
||||||
mkdir -p "$out"
|
|
||||||
unzip -q "$zip" -d "$out"
|
unzip -q "$zip" -d "$out"
|
||||||
echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out"
|
echo "nightly-fetch: $(find "$out" -type f | wc -l) files in $out"
|
||||||
Executable
+19
@@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Runs rclone for the publish job and says when it only got there on a retry.
|
||||||
|
#
|
||||||
|
# nightly-rclone <rclone arguments...>
|
||||||
|
#
|
||||||
|
# rclone tries a failed transfer again by itself, and a run that passes on the
|
||||||
|
# second attempt looks like any other. This passes rclone's output and exit
|
||||||
|
# status through, and adds a warning for the run's page naming the files that
|
||||||
|
# failed on an earlier attempt.
|
||||||
|
set -uo pipefail
|
||||||
|
log=$(mktemp)
|
||||||
|
trap 'rm -f "$log"' EXIT
|
||||||
|
rclone "$@" 2>&1 | tee "$log"
|
||||||
|
status=${PIPESTATUS[0]}
|
||||||
|
files=$(sed -nE 's/^.*ERROR : (.*): Failed to copy: .*$/\1/p' "$log" | sort -u)
|
||||||
|
if [[ $status -eq 0 && -n $files ]]; then
|
||||||
|
echo "::warning::rclone needed a retry for $(wc -l <<<"$files") file(s): $(paste -sd' ' <<<"$files")"
|
||||||
|
fi
|
||||||
|
exit "$status"
|
||||||
Reference in new issue
Block a user