From ed0dfc50529b78f6e32c98b96c2f43299d9f6e4d Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Fri, 9 Oct 2026 22:36:31 +0200 Subject: [PATCH] Nightly: the retry warning covers latest.json and counts files The warning was written for the bulk upload only, and counted lines: latest.json is uploaded by a second rclone call, which could still pass on a retry unseen, and a file that failed on two attempts was counted as two uploads. Both calls go through test/vm-image/nightly-rclone, which passes rclone's output and exit status through and names each file once. A stand-in rclone tests it. --- .github/workflows/nightly-build.yml | 16 ++++++---------- test/unit/nightly-test.sh | 27 +++++++++++++++++++++++++++ test/vm-image/nightly-rclone | 19 +++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100755 test/vm-image/nightly-rclone diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index d109201..5bd0333 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -285,17 +285,13 @@ jobs: # --immutable: a name that exists with other bytes is an error, not an # overwrite. - # An upload that only passes on a retry is said so on the run's page: - # the files are compared afterwards either way, but it should not go - # unnoticed again. + # nightly-rclone is rclone, with a warning on the run's page when an + # upload only passed on a retry: the files are compared afterwards either + # way, but it should not go unnoticed again. - name: Upload this version run: | - set -euo pipefail - rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \ - --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line 2>&1 | tee upload.log - if grep -q "Failed to copy" upload.log; then - echo "::warning::rclone needed a retry: $(grep -c "Failed to copy" upload.log) uploads failed on an earlier attempt" - fi + ./test/vm-image/nightly-rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \ + --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line # Fetched the way a user fetches them. It proves what is served, and it # is the first request the CDN sees for each file. @@ -318,7 +314,7 @@ jobs: - name: Point latest.json at it run: | - rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket + ./test/vm-image/nightly-rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version) [ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; } echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY" diff --git a/test/unit/nightly-test.sh b/test/unit/nightly-test.sh index f17bd5d..ea4ff23 100644 --- a/test/unit/nightly-test.sh +++ b/test/unit/nightly-test.sh @@ -187,6 +187,33 @@ else echo "nightly-fetch tests skipped: no zip or unzip here" fi +# -------------------------------------------------------------- nightly-rclone +wrapped="$ROOT/test/vm-image/nightly-rclone" +bash -n "$wrapped" +mkdir -p "$tmp/rc" +rclone_says() { # exit status, output lines...: a stand-in rclone + local status=$1; shift + { echo '#!/bin/bash'; printf 'echo %q >&2\n' "$@"; echo "exit $status"; } >"$tmp/rc/rclone" + chmod +x "$tmp/rc/rclone" +} +rclone_says 0 "Transferred: 3 / 3, 100%" +out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b) +[[ $out != *::warning::* ]] || fail "rclone: a clean upload was reported as a retry" +[[ $out == *"Transferred: 3 / 3"* ]] || fail "rclone: its output was not passed through" + +rclone_says 0 "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \ + "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \ + "<3>ERROR : unpacked/vmlinuz: Failed to copy: NotImplemented: Not Implemented" \ + "<3>ERROR : Attempt 2/3 failed with 2 errors" "<3>ERROR : Attempt 3/3 succeeded" +out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b) +[[ $out == *"::warning::rclone needed a retry for 2 file(s): a.qcow2 unpacked/vmlinuz"* ]] || + fail "rclone: a file that failed twice is not counted once: $(grep warning <<<"$out")" + +rclone_says 3 "<3>ERROR : a.qcow2: Failed to copy: AccessDenied" +if PATH="$tmp/rc:$PATH" "$wrapped" copy a b >"$tmp/rc/out" 2>&1; then fail "rclone: a failed upload passed"; fi +equal "rclone: its exit status is kept" "$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b >/dev/null 2>&1; echo $?)" "3" +if grep -q '::warning::' "$tmp/rc/out"; then fail "rclone: a failed upload was reported as a retry"; fi + # ------------------------------------------------------------ the install test # The harness is a script that starts a VM when it is read; take the two # functions out of it. diff --git a/test/vm-image/nightly-rclone b/test/vm-image/nightly-rclone new file mode 100755 index 0000000..a9b9eb7 --- /dev/null +++ b/test/vm-image/nightly-rclone @@ -0,0 +1,19 @@ +#!/bin/bash +# Runs rclone for the publish job and says when it only got there on a retry. +# +# nightly-rclone +# +# rclone tries a failed transfer again by itself, and a run that passes on the +# second attempt looks like any other. This passes rclone's output and exit +# status through, and adds a warning for the run's page naming the files that +# failed on an earlier attempt. +set -uo pipefail +log=$(mktemp) +trap 'rm -f "$log"' EXIT +rclone "$@" 2>&1 | tee "$log" +status=${PIPESTATUS[0]} +files=$(sed -nE 's/^.*ERROR : (.*): Failed to copy: .*$/\1/p' "$log" | sort -u) +if [[ $status -eq 0 && -n $files ]]; then + echo "::warning::rclone needed a retry for $(wc -l <<<"$files") file(s): $(paste -sd' ' <<<"$files")" +fi +exit "$status"