Prune unneeded packages from the ISO mirror cache
This commit is contained in:
1 parent
17532793fd
commit
f66b6fc28e
3 files changed
+185
-38
No files matched your search
+39
-38
@@ -201,48 +201,49 @@ if ! download_offline_packages; then
|
|||||||
download_offline_packages
|
download_offline_packages
|
||||||
fi
|
fi
|
||||||
|
|
||||||
prune_stale_package_versions() {
|
# Resolve the exact filenames chosen by the same synced package databases used
|
||||||
local dir="$1"
|
# for the download. Pruning by this transaction (rather than merely keeping the
|
||||||
local pkgfile name version cmp
|
# newest version of every cached package name) removes packages that have left
|
||||||
local -a stale=()
|
# the lists or dependency closure, such as an old Electron major version.
|
||||||
declare -A newest_file=()
|
if ! resolved_package_files="$(
|
||||||
declare -A newest_version=()
|
pacman --config "/configs/pacman-online-${OMARCHY_MIRROR}.conf" --noconfirm \
|
||||||
|
--dbpath /tmp/offlinedb -S --print --print-format '%f' "${all_packages[@]}"
|
||||||
|
)"; then
|
||||||
|
echo "ERROR: could not resolve the package files required by the offline mirror" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mapfile -t required_package_files <<< "$resolved_package_files"
|
||||||
|
|
||||||
while IFS= read -r pkgfile; do
|
# The online transaction intentionally excludes packages built from the local
|
||||||
read -r name version < <(pacman -Qp "$pkgfile" 2>/dev/null) || continue
|
# checkouts. Add those exact artifacts back to the keep-set after verifying
|
||||||
[[ -n $name && -n $version ]] || continue
|
# that the local build left exactly one file for each selected package name.
|
||||||
|
if [[ -n ${LOCAL_OMARCHY_BUILD:-} ]]; then
|
||||||
if [[ -z ${newest_version[$name]+x} ]]; then
|
for local_package_name in \
|
||||||
newest_version[$name]="$version"
|
"$OMARCHY_RUNTIME_PACKAGE" "$OMARCHY_SETTINGS_PACKAGE" "$OMARCHY_NVIM_PACKAGE"; do
|
||||||
newest_file[$name]="$pkgfile"
|
local_package_file=""
|
||||||
continue
|
for candidate in "$offline_mirror_dir/$local_package_name-"*.pkg.tar.*; do
|
||||||
fi
|
[[ -f $candidate && $candidate != *.sig ]] || continue
|
||||||
|
read -r candidate_name _ < <(pacman -Qp "$candidate" 2>/dev/null) || continue
|
||||||
cmp=$(vercmp "$version" "${newest_version[$name]}")
|
[[ $candidate_name == "$local_package_name" ]] || continue
|
||||||
if (( cmp > 0 )); then
|
if [[ -n $local_package_file ]]; then
|
||||||
stale+=("${newest_file[$name]}")
|
echo "ERROR: multiple local builds found for $local_package_name" >&2
|
||||||
newest_version[$name]="$version"
|
exit 1
|
||||||
newest_file[$name]="$pkgfile"
|
fi
|
||||||
else
|
local_package_file="${candidate##*/}"
|
||||||
stale+=("$pkgfile")
|
|
||||||
fi
|
|
||||||
done < <(find "$dir" -maxdepth 1 -type f -name '*.pkg.tar.*' ! -name '*.sig' -print | sort)
|
|
||||||
|
|
||||||
if (( ${#stale[@]} > 0 )); then
|
|
||||||
echo "Pruning stale package versions from offline mirror:"
|
|
||||||
for pkgfile in "${stale[@]}"; do
|
|
||||||
echo " $(basename "$pkgfile")"
|
|
||||||
rm -f "$pkgfile" "$pkgfile.sig"
|
|
||||||
done
|
done
|
||||||
fi
|
if [[ -z $local_package_file ]]; then
|
||||||
}
|
echo "ERROR: local build not found for $local_package_name" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
required_package_files+=("$local_package_file")
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '%s\n' "${required_package_files[@]}" |
|
||||||
|
bash /builder/prune-offline-mirror.sh "$offline_mirror_dir"
|
||||||
|
|
||||||
# Rebuild the offline repo db from scratch so size/checksum/depends entries
|
# Rebuild the offline repo db from scratch so size/checksum/depends entries
|
||||||
# always reflect the current package files. The persistent cache can contain
|
# always reflect only the package files selected for this build.
|
||||||
# multiple versions of the same package; repo-add keeps the last one it sees,
|
|
||||||
# which is lexical glob order, not version order (e.g. pkgrel -9 can override
|
|
||||||
# -15). Prune to one newest version per package before indexing.
|
|
||||||
prune_stale_package_versions "$offline_mirror_dir"
|
|
||||||
rm -f "$offline_mirror_dir"/offline.db* "$offline_mirror_dir"/offline.files*
|
rm -f "$offline_mirror_dir"/offline.db* "$offline_mirror_dir"/offline.files*
|
||||||
repo-add "$offline_mirror_dir/offline.db.tar.gz" "$offline_mirror_dir/"*.pkg.tar.zst
|
repo-add "$offline_mirror_dir/offline.db.tar.gz" "$offline_mirror_dir/"*.pkg.tar.zst
|
||||||
|
|
||||||
|
|||||||
Executable
+67
@@ -0,0 +1,67 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
offline_mirror_dir="${1:-}"
|
||||||
|
if [[ -z $offline_mirror_dir ]]; then
|
||||||
|
echo "Usage: prune-offline-mirror.sh <offline-mirror-dir>" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ ! -d $offline_mirror_dir ]]; then
|
||||||
|
echo "ERROR: offline mirror directory not found: $offline_mirror_dir" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Read the exact package filenames selected by pacman on stdin. Validate the
|
||||||
|
# complete selection before deleting anything: an empty/partial resolution must
|
||||||
|
# never turn a warm-cache build into a broken mirror.
|
||||||
|
declare -A required=()
|
||||||
|
while IFS= read -r filename; do
|
||||||
|
[[ -n $filename ]] || continue
|
||||||
|
if [[ $filename == */* || $filename != *.pkg.tar.* || $filename == *.sig ]]; then
|
||||||
|
echo "ERROR: invalid required package filename: $filename" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
required["$filename"]=1
|
||||||
|
done
|
||||||
|
|
||||||
|
if (( ${#required[@]} == 0 )); then
|
||||||
|
echo "ERROR: refusing to prune the offline mirror with an empty package selection" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
missing=()
|
||||||
|
for filename in "${!required[@]}"; do
|
||||||
|
[[ -f "$offline_mirror_dir/$filename" ]] || missing+=("$filename")
|
||||||
|
done
|
||||||
|
if (( ${#missing[@]} > 0 )); then
|
||||||
|
echo "ERROR: refusing to prune; selected packages are missing from the mirror:" >&2
|
||||||
|
printf ' %s\n' "${missing[@]}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
stale=()
|
||||||
|
while IFS= read -r -d '' package_file; do
|
||||||
|
filename="${package_file##*/}"
|
||||||
|
[[ -n ${required[$filename]+x} ]] || stale+=("$package_file")
|
||||||
|
done < <(
|
||||||
|
find "$offline_mirror_dir" -maxdepth 1 -type f \
|
||||||
|
-name '*.pkg.tar.*' ! -name '*.sig' -print0
|
||||||
|
)
|
||||||
|
|
||||||
|
if (( ${#stale[@]} > 0 )); then
|
||||||
|
echo "Pruning package archives not selected for the offline mirror:"
|
||||||
|
for package_file in "${stale[@]}"; do
|
||||||
|
echo " $(basename "$package_file")"
|
||||||
|
rm -f -- "$package_file" "$package_file.sig"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
# A failed or interrupted download can leave a signature without its archive.
|
||||||
|
# It cannot be indexed and only wastes space in the ISO.
|
||||||
|
while IFS= read -r -d '' signature_file; do
|
||||||
|
[[ -f ${signature_file%.sig} ]] || rm -f -- "$signature_file"
|
||||||
|
done < <(
|
||||||
|
find "$offline_mirror_dir" -maxdepth 1 -type f \
|
||||||
|
-name '*.pkg.tar.*.sig' -print0
|
||||||
|
)
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
PRUNER = ROOT / "builder/prune-offline-mirror.sh"
|
||||||
|
|
||||||
|
|
||||||
|
class OfflineMirrorPruningTest(unittest.TestCase):
|
||||||
|
def run_pruner(self, mirror: Path, required: list[str]):
|
||||||
|
return subprocess.run(
|
||||||
|
["bash", str(PRUNER), str(mirror)],
|
||||||
|
input="".join(f"{filename}\n" for filename in required),
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_file(self, mirror: Path, filename: str):
|
||||||
|
(mirror / filename).write_bytes(b"package")
|
||||||
|
|
||||||
|
def test_keeps_only_exactly_selected_package_archives(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
mirror = Path(directory)
|
||||||
|
keep = "keep-2.0-1-x86_64.pkg.tar.zst"
|
||||||
|
other = "other-1.0-1-any.pkg.tar.zst"
|
||||||
|
stale_version = "keep-1.0-1-x86_64.pkg.tar.zst"
|
||||||
|
removed_package = "removed-1.0-1-x86_64.pkg.tar.zst"
|
||||||
|
orphan_signature = "interrupted-1.0-1-x86_64.pkg.tar.zst.sig"
|
||||||
|
|
||||||
|
for filename in (keep, other, stale_version, removed_package):
|
||||||
|
self.add_file(mirror, filename)
|
||||||
|
for filename in (keep, stale_version, removed_package):
|
||||||
|
self.add_file(mirror, f"{filename}.sig")
|
||||||
|
self.add_file(mirror, orphan_signature)
|
||||||
|
self.add_file(mirror, "offline.db.tar.gz")
|
||||||
|
|
||||||
|
result = self.run_pruner(mirror, [keep, other])
|
||||||
|
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertEqual(
|
||||||
|
{path.name for path in mirror.iterdir()},
|
||||||
|
{keep, f"{keep}.sig", other, "offline.db.tar.gz"},
|
||||||
|
)
|
||||||
|
self.assertIn(stale_version, result.stdout)
|
||||||
|
self.assertIn(removed_package, result.stdout)
|
||||||
|
|
||||||
|
def test_missing_selection_aborts_before_deleting_anything(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
mirror = Path(directory)
|
||||||
|
cached = "cached-1.0-1-x86_64.pkg.tar.zst"
|
||||||
|
self.add_file(mirror, cached)
|
||||||
|
|
||||||
|
result = self.run_pruner(
|
||||||
|
mirror, ["missing-1.0-1-x86_64.pkg.tar.zst"]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertTrue((mirror / cached).exists())
|
||||||
|
self.assertIn("selected packages are missing", result.stderr)
|
||||||
|
|
||||||
|
def test_empty_selection_aborts_before_deleting_anything(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
mirror = Path(directory)
|
||||||
|
cached = "cached-1.0-1-x86_64.pkg.tar.zst"
|
||||||
|
self.add_file(mirror, cached)
|
||||||
|
|
||||||
|
result = self.run_pruner(mirror, [])
|
||||||
|
|
||||||
|
self.assertNotEqual(result.returncode, 0)
|
||||||
|
self.assertTrue((mirror / cached).exists())
|
||||||
|
self.assertIn("empty package selection", result.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in new issue
Block a user