diff --git a/builder/build-iso.sh b/builder/build-iso.sh index b52a5b1..ae7719b 100755 --- a/builder/build-iso.sh +++ b/builder/build-iso.sh @@ -201,48 +201,49 @@ if ! download_offline_packages; then download_offline_packages fi -prune_stale_package_versions() { - local dir="$1" - local pkgfile name version cmp - local -a stale=() - declare -A newest_file=() - declare -A newest_version=() +# Resolve the exact filenames chosen by the same synced package databases used +# for the download. Pruning by this transaction (rather than merely keeping the +# newest version of every cached package name) removes packages that have left +# the lists or dependency closure, such as an old Electron major version. +if ! resolved_package_files="$( + pacman --config "/configs/pacman-online-${OMARCHY_MIRROR}.conf" --noconfirm \ + --dbpath /tmp/offlinedb -S --print --print-format '%f' "${all_packages[@]}" +)"; then + echo "ERROR: could not resolve the package files required by the offline mirror" >&2 + exit 1 +fi +mapfile -t required_package_files <<< "$resolved_package_files" - while IFS= read -r pkgfile; do - read -r name version < <(pacman -Qp "$pkgfile" 2>/dev/null) || continue - [[ -n $name && -n $version ]] || continue - - if [[ -z ${newest_version[$name]+x} ]]; then - newest_version[$name]="$version" - newest_file[$name]="$pkgfile" - continue - fi - - cmp=$(vercmp "$version" "${newest_version[$name]}") - if (( cmp > 0 )); then - stale+=("${newest_file[$name]}") - newest_version[$name]="$version" - newest_file[$name]="$pkgfile" - else - stale+=("$pkgfile") - fi - done < <(find "$dir" -maxdepth 1 -type f -name '*.pkg.tar.*' ! -name '*.sig' -print | sort) - - if (( ${#stale[@]} > 0 )); then - echo "Pruning stale package versions from offline mirror:" - for pkgfile in "${stale[@]}"; do - echo " $(basename "$pkgfile")" - rm -f "$pkgfile" "$pkgfile.sig" +# The online transaction intentionally excludes packages built from the local +# checkouts. Add those exact artifacts back to the keep-set after verifying +# that the local build left exactly one file for each selected package name. +if [[ -n ${LOCAL_OMARCHY_BUILD:-} ]]; then + for local_package_name in \ + "$OMARCHY_RUNTIME_PACKAGE" "$OMARCHY_SETTINGS_PACKAGE" "$OMARCHY_NVIM_PACKAGE"; do + local_package_file="" + for candidate in "$offline_mirror_dir/$local_package_name-"*.pkg.tar.*; do + [[ -f $candidate && $candidate != *.sig ]] || continue + read -r candidate_name _ < <(pacman -Qp "$candidate" 2>/dev/null) || continue + [[ $candidate_name == "$local_package_name" ]] || continue + if [[ -n $local_package_file ]]; then + echo "ERROR: multiple local builds found for $local_package_name" >&2 + exit 1 + fi + local_package_file="${candidate##*/}" done - fi -} + if [[ -z $local_package_file ]]; then + echo "ERROR: local build not found for $local_package_name" >&2 + exit 1 + fi + required_package_files+=("$local_package_file") + done +fi + +printf '%s\n' "${required_package_files[@]}" | + bash /builder/prune-offline-mirror.sh "$offline_mirror_dir" # Rebuild the offline repo db from scratch so size/checksum/depends entries -# always reflect the current package files. The persistent cache can contain -# multiple versions of the same package; repo-add keeps the last one it sees, -# which is lexical glob order, not version order (e.g. pkgrel -9 can override -# -15). Prune to one newest version per package before indexing. -prune_stale_package_versions "$offline_mirror_dir" +# always reflect only the package files selected for this build. rm -f "$offline_mirror_dir"/offline.db* "$offline_mirror_dir"/offline.files* repo-add "$offline_mirror_dir/offline.db.tar.gz" "$offline_mirror_dir/"*.pkg.tar.zst diff --git a/builder/prune-offline-mirror.sh b/builder/prune-offline-mirror.sh new file mode 100755 index 0000000..ba43119 --- /dev/null +++ b/builder/prune-offline-mirror.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -euo pipefail + +offline_mirror_dir="${1:-}" +if [[ -z $offline_mirror_dir ]]; then + echo "Usage: prune-offline-mirror.sh " >&2 + exit 1 +fi +if [[ ! -d $offline_mirror_dir ]]; then + echo "ERROR: offline mirror directory not found: $offline_mirror_dir" >&2 + exit 1 +fi + +# Read the exact package filenames selected by pacman on stdin. Validate the +# complete selection before deleting anything: an empty/partial resolution must +# never turn a warm-cache build into a broken mirror. +declare -A required=() +while IFS= read -r filename; do + [[ -n $filename ]] || continue + if [[ $filename == */* || $filename != *.pkg.tar.* || $filename == *.sig ]]; then + echo "ERROR: invalid required package filename: $filename" >&2 + exit 1 + fi + required["$filename"]=1 +done + +if (( ${#required[@]} == 0 )); then + echo "ERROR: refusing to prune the offline mirror with an empty package selection" >&2 + exit 1 +fi + +missing=() +for filename in "${!required[@]}"; do + [[ -f "$offline_mirror_dir/$filename" ]] || missing+=("$filename") +done +if (( ${#missing[@]} > 0 )); then + echo "ERROR: refusing to prune; selected packages are missing from the mirror:" >&2 + printf ' %s\n' "${missing[@]}" >&2 + exit 1 +fi + +stale=() +while IFS= read -r -d '' package_file; do + filename="${package_file##*/}" + [[ -n ${required[$filename]+x} ]] || stale+=("$package_file") +done < <( + find "$offline_mirror_dir" -maxdepth 1 -type f \ + -name '*.pkg.tar.*' ! -name '*.sig' -print0 +) + +if (( ${#stale[@]} > 0 )); then + echo "Pruning package archives not selected for the offline mirror:" + for package_file in "${stale[@]}"; do + echo " $(basename "$package_file")" + rm -f -- "$package_file" "$package_file.sig" + done +fi + +# A failed or interrupted download can leave a signature without its archive. +# It cannot be indexed and only wastes space in the ISO. +while IFS= read -r -d '' signature_file; do + [[ -f ${signature_file%.sig} ]] || rm -f -- "$signature_file" +done < <( + find "$offline_mirror_dir" -maxdepth 1 -type f \ + -name '*.pkg.tar.*.sig' -print0 +) diff --git a/test/test_offline_mirror_pruning.py b/test/test_offline_mirror_pruning.py new file mode 100644 index 0000000..88ed468 --- /dev/null +++ b/test/test_offline_mirror_pruning.py @@ -0,0 +1,79 @@ +#!/usr/bin/python + +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PRUNER = ROOT / "builder/prune-offline-mirror.sh" + + +class OfflineMirrorPruningTest(unittest.TestCase): + def run_pruner(self, mirror: Path, required: list[str]): + return subprocess.run( + ["bash", str(PRUNER), str(mirror)], + input="".join(f"{filename}\n" for filename in required), + capture_output=True, + text=True, + ) + + def add_file(self, mirror: Path, filename: str): + (mirror / filename).write_bytes(b"package") + + def test_keeps_only_exactly_selected_package_archives(self): + with tempfile.TemporaryDirectory() as directory: + mirror = Path(directory) + keep = "keep-2.0-1-x86_64.pkg.tar.zst" + other = "other-1.0-1-any.pkg.tar.zst" + stale_version = "keep-1.0-1-x86_64.pkg.tar.zst" + removed_package = "removed-1.0-1-x86_64.pkg.tar.zst" + orphan_signature = "interrupted-1.0-1-x86_64.pkg.tar.zst.sig" + + for filename in (keep, other, stale_version, removed_package): + self.add_file(mirror, filename) + for filename in (keep, stale_version, removed_package): + self.add_file(mirror, f"{filename}.sig") + self.add_file(mirror, orphan_signature) + self.add_file(mirror, "offline.db.tar.gz") + + result = self.run_pruner(mirror, [keep, other]) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + {path.name for path in mirror.iterdir()}, + {keep, f"{keep}.sig", other, "offline.db.tar.gz"}, + ) + self.assertIn(stale_version, result.stdout) + self.assertIn(removed_package, result.stdout) + + def test_missing_selection_aborts_before_deleting_anything(self): + with tempfile.TemporaryDirectory() as directory: + mirror = Path(directory) + cached = "cached-1.0-1-x86_64.pkg.tar.zst" + self.add_file(mirror, cached) + + result = self.run_pruner( + mirror, ["missing-1.0-1-x86_64.pkg.tar.zst"] + ) + + self.assertNotEqual(result.returncode, 0) + self.assertTrue((mirror / cached).exists()) + self.assertIn("selected packages are missing", result.stderr) + + def test_empty_selection_aborts_before_deleting_anything(self): + with tempfile.TemporaryDirectory() as directory: + mirror = Path(directory) + cached = "cached-1.0-1-x86_64.pkg.tar.zst" + self.add_file(mirror, cached) + + result = self.run_pruner(mirror, []) + + self.assertNotEqual(result.returncode, 0) + self.assertTrue((mirror / cached).exists()) + self.assertIn("empty package selection", result.stderr) + + +if __name__ == "__main__": + unittest.main()