Nightly: the retry warning covers latest.json and counts files

The warning was written for the bulk upload only, and counted lines:
latest.json is uploaded by a second rclone call, which could still pass
on a retry unseen, and a file that failed on two attempts was counted as
two uploads.

Both calls go through test/vm-image/nightly-rclone, which passes
rclone's output and exit status through and names each file once. A
stand-in rclone tests it.

.github/workflows/nightly-build.yml

@@ -285,17 +285,13 @@ jobs:
 
       # --immutable: a name that exists with other bytes is an error, not an
       # overwrite.
-      # An upload that only passes on a retry is said so on the run's page:
-      # the files are compared afterwards either way, but it should not go
-      # unnoticed again.
+      # nightly-rclone is rclone, with a warning on the run's page when an
+      # upload only passed on a retry: the files are compared afterwards either
+      # way, but it should not go unnoticed again.
       - name: Upload this version
         run: |
-          set -euo pipefail
-          rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
-            --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line 2>&1 | tee upload.log
-          if grep -q "Failed to copy" upload.log; then
-            echo "::warning::rclone needed a retry: $(grep -c "Failed to copy" upload.log) uploads failed on an earlier attempt"
-          fi
+          ./test/vm-image/nightly-rclone copy "out/$RELEASE" "r2:$BUCKET/$RELEASE" \
+            --immutable --s3-no-check-bucket --transfers 8 --s3-upload-concurrency 8 --s3-chunk-size 64M --stats 30s --stats-one-line
 
       # Fetched the way a user fetches them. It proves what is served, and it
       # is the first request the CDN sees for each file.
@@ -318,7 +314,7 @@ jobs:
 
       - name: Point latest.json at it
         run: |
-          rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
+          ./test/vm-image/nightly-rclone copyto "out/$CHANNEL/latest.json" "r2:$BUCKET/$CHANNEL/latest.json" --s3-no-check-bucket
           served=$(curl -fsSL "$SITE/$CHANNEL/latest.json?run=$GITHUB_RUN_ID" | jq -r .version)
           [ "$served" = "$VERSION" ] || { echo "::error::latest.json serves version $served, expected $VERSION"; exit 1; }
           echo "Published [$VERSION]($SITE/$CHANNEL/latest.json)" >>"$GITHUB_STEP_SUMMARY"

test/unit/nightly-test.sh

@@ -187,6 +187,33 @@ else
   echo "nightly-fetch tests skipped: no zip or unzip here"
 fi
 
+# -------------------------------------------------------------- nightly-rclone
+wrapped="$ROOT/test/vm-image/nightly-rclone"
+bash -n "$wrapped"
+mkdir -p "$tmp/rc"
+rclone_says() { # exit status, output lines...: a stand-in rclone
+  local status=$1; shift
+  { echo '#!/bin/bash'; printf 'echo %q >&2\n' "$@"; echo "exit $status"; } >"$tmp/rc/rclone"
+  chmod +x "$tmp/rc/rclone"
+}
+rclone_says 0 "Transferred: 3 / 3, 100%"
+out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b)
+[[ $out != *::warning::* ]] || fail "rclone: a clean upload was reported as a retry"
+[[ $out == *"Transferred: 3 / 3"* ]] || fail "rclone: its output was not passed through"
+
+rclone_says 0 "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \
+  "<3>ERROR : a.qcow2: Failed to copy: NotImplemented: Not Implemented" \
+  "<3>ERROR : unpacked/vmlinuz: Failed to copy: NotImplemented: Not Implemented" \
+  "<3>ERROR : Attempt 2/3 failed with 2 errors" "<3>ERROR : Attempt 3/3 succeeded"
+out=$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b)
+[[ $out == *"::warning::rclone needed a retry for 2 file(s): a.qcow2 unpacked/vmlinuz"* ]] ||
+  fail "rclone: a file that failed twice is not counted once: $(grep warning <<<"$out")"
+
+rclone_says 3 "<3>ERROR : a.qcow2: Failed to copy: AccessDenied"
+if PATH="$tmp/rc:$PATH" "$wrapped" copy a b >"$tmp/rc/out" 2>&1; then fail "rclone: a failed upload passed"; fi
+equal "rclone: its exit status is kept" "$(PATH="$tmp/rc:$PATH" "$wrapped" copy a b >/dev/null 2>&1; echo $?)" "3"
+if grep -q '::warning::' "$tmp/rc/out"; then fail "rclone: a failed upload was reported as a retry"; fi
+
 # ------------------------------------------------------------ the install test
 # The harness is a script that starts a VM when it is read; take the two
 # functions out of it.

test/vm-image/nightly-rclone

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Runs rclone for the publish job and says when it only got there on a retry.
+#
+#   nightly-rclone <rclone arguments...>
+#
+# rclone tries a failed transfer again by itself, and a run that passes on the
+# second attempt looks like any other. This passes rclone's output and exit
+# status through, and adds a warning for the run's page naming the files that
+# failed on an earlier attempt.
+set -uo pipefail
+log=$(mktemp)
+trap 'rm -f "$log"' EXIT
+rclone "$@" 2>&1 | tee "$log"
+status=${PIPESTATUS[0]}
+files=$(sed -nE 's/^.*ERROR : (.*): Failed to copy: .*$/\1/p' "$log" | sort -u)
+if [[ $status -eq 0 && -n $files ]]; then
+  echo "::warning::rclone needed a retry for $(wc -l <<<"$files") file(s): $(paste -sd' ' <<<"$files")"
+fi
+exit "$status"