Reuse existing build artifacts when a PR's package tree is unchanged
Every push to a PR rebuilt every package the PR touches, on every architecture, even when only one of them changed. The daily sync PR carries around thirty package/arch pairs; fixing one package meant rebuilding all of them, and an aarch64 build under QEMU takes up to an hour. Nine runs of that PR cost about 36 droplet-hours in two days. The planner now asks the artifact store for <pkg>-<arch>-<tree hash> before adding an entry to the matrix and drops entries that already have one. That is the same lookup publish.yml makes on merge, so a reused entry publishes exactly the file it would have anyway. Dry run against the current sync PR: 27 of 31 entries reused, 4 built. Pack and Upload no longer run with always(): only a successful build uploads, so an artifact's existence means that tree built. workflow_dispatch always builds; it is an explicit request.
This commit is contained in:
1 parent
b7706e8f62
commit
c74c708f35
1 file changed
+29
-3
@@ -81,6 +81,8 @@ jobs:
|
|||||||
# once, against edge; the channels it ships to on merge are carried
|
# once, against edge; the channels it ships to on merge are carried
|
||||||
# along for information. A filename means one set of bytes.
|
# along for information. A filename means one set of bytes.
|
||||||
- id: list
|
- id: list
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
if [[ -n "${{ github.event.inputs.packages }}" ]]; then
|
if [[ -n "${{ github.event.inputs.packages }}" ]]; then
|
||||||
names="${{ github.event.inputs.packages }}"
|
names="${{ github.event.inputs.packages }}"
|
||||||
@@ -89,6 +91,29 @@ jobs:
|
|||||||
| awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u)
|
| awk -F/ '$1=="pkgbuilds" && NF>2 {print $2}' | sort -u)
|
||||||
fi
|
fi
|
||||||
matrix=$(printf '%s\n' $names | bin/build-matrix)
|
matrix=$(printf '%s\n' $names | bin/build-matrix)
|
||||||
|
# A package directory whose exact tree already has a build artifact
|
||||||
|
# (label <pkg>-<arch>-<tree hash>, uploaded only after a successful
|
||||||
|
# build) is not built again. Pushing a fix for one package to a PR
|
||||||
|
# that touches fifty rebuilds one, not fifty; publish.yml finds the
|
||||||
|
# same artifacts on merge. workflow_dispatch is an explicit request
|
||||||
|
# and always builds.
|
||||||
|
if [[ "${{ github.event_name }}" == pull_request ]]; then
|
||||||
|
head="${{ github.event.pull_request.head.sha }}"
|
||||||
|
kept=(); reused=()
|
||||||
|
while read -r entry; do
|
||||||
|
package=$(jq -r .package <<<"$entry"); arch=$(jq -r .arch <<<"$entry")
|
||||||
|
label="$package-$arch-$(git rev-parse "$head:pkgbuilds/$package")"
|
||||||
|
found=$(curl -fsS -H "Authorization: Bearer $GH_TOKEN" -H "Accept: application/vnd.github+json" \
|
||||||
|
"https://api.github.com/repos/${{ github.repository }}/actions/artifacts?name=$label&per_page=5" \
|
||||||
|
| jq -r '[.artifacts[] | select(.expired|not)] | length' || echo 0)
|
||||||
|
if (( found > 0 )); then reused+=("$label"); else kept+=("$entry"); fi
|
||||||
|
done < <(jq -c '.include[]' <<<"$matrix")
|
||||||
|
matrix=$(printf '%s\n' "${kept[@]}" | jq -sc '{include: .}')
|
||||||
|
if (( ${#reused[@]} )); then
|
||||||
|
printf '==> already built, reusing the artifact: %s\n' "${reused[@]}"
|
||||||
|
{ echo "Reused existing build artifacts (${#reused[@]}):"; printf -- '- %s\n' "${reused[@]}"; } >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
||||||
echo "planned=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT"
|
echo "planned=$(jq '.include | length' <<<"$matrix")" >> "$GITHUB_OUTPUT"
|
||||||
jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix"
|
jq -r '.include[] | "\(.package) \(.arch) -> \(.channels)"' <<<"$matrix"
|
||||||
@@ -163,6 +188,7 @@ jobs:
|
|||||||
echo "tooling: $(git rev-parse --short HEAD) (base) packages: ${{ github.event.pull_request.head.sha }} (PR head)"
|
echo "tooling: $(git rev-parse --short HEAD) (base) packages: ${{ github.event.pull_request.head.sha }} (PR head)"
|
||||||
git status --short | head
|
git status --short | head
|
||||||
- name: Build ${{ matrix.package }} (${{ matrix.arch }}, ships to ${{ matrix.channels }})
|
- name: Build ${{ matrix.package }} (${{ matrix.arch }}, ships to ${{ matrix.channels }})
|
||||||
|
id: build
|
||||||
env:
|
env:
|
||||||
CONTAINER_ENGINE: docker
|
CONTAINER_ENGINE: docker
|
||||||
run: bin/build --mirror edge --arch ${{ matrix.arch }} --package ${{ matrix.package }}
|
run: bin/build --mirror edge --arch ${{ matrix.arch }} --package ${{ matrix.package }}
|
||||||
@@ -176,16 +202,16 @@ jobs:
|
|||||||
run: echo "hash=$(git rev-parse "${{ github.event.pull_request.head.sha || github.sha }}:pkgbuilds/${{ matrix.package }}")" >> "$GITHUB_OUTPUT"
|
run: echo "hash=$(git rev-parse "${{ github.event.pull_request.head.sha || github.sha }}:pkgbuilds/${{ matrix.package }}")" >> "$GITHUB_OUTPUT"
|
||||||
# The upload action rejects a path containing ':', which is how makepkg
|
# The upload action rejects a path containing ':', which is how makepkg
|
||||||
# names a package with an epoch. The files ride inside packages.tar
|
# names a package with an epoch. The files ride inside packages.tar
|
||||||
# (helpers/artifact-helpers.sh); publish.yml unpacks it.
|
# (helpers/artifact-helpers.sh); publish.yml unpacks it. Only a
|
||||||
|
# successful build uploads: the artifact's existence is what lets the
|
||||||
|
# planner above and publish.yml skip rebuilding this exact tree.
|
||||||
- name: Pack artifact
|
- name: Pack artifact
|
||||||
id: pack
|
id: pack
|
||||||
if: always()
|
|
||||||
run: |
|
run: |
|
||||||
source helpers/artifact-helpers.sh
|
source helpers/artifact-helpers.sh
|
||||||
pack_packages build-output/edge/${{ matrix.arch }} packages.tar
|
pack_packages build-output/edge/${{ matrix.arch }} packages.tar
|
||||||
tar -tvf packages.tar
|
tar -tvf packages.tar
|
||||||
- name: Upload artifact
|
- name: Upload artifact
|
||||||
if: always() && steps.pack.outcome == 'success'
|
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: ${{ matrix.package }}-${{ matrix.arch }}-${{ steps.tree.outputs.hash }}
|
name: ${{ matrix.package }}-${{ matrix.arch }}-${{ steps.tree.outputs.hash }}
|
||||||
|
|||||||
Reference in new issue
Block a user