diff --git a/builder/build-iso.sh b/builder/build-iso.sh index 8b444fe..13a90d9 100755 --- a/builder/build-iso.sh +++ b/builder/build-iso.sh @@ -304,8 +304,9 @@ printf '%s\n' "${required_package_files[@]}" | # Rebuild the offline repo db from scratch so size/checksum/depends entries # always reflect only the package files selected for this build. -rm -f "$offline_mirror_dir"/offline.db* "$offline_mirror_dir"/offline.files* -repo-add "$offline_mirror_dir/offline.db.tar.gz" "$offline_mirror_dir/"*.pkg.tar.zst +# Every package archive in the mirror, indexed by one repo-add per CPU: one +# alone hashes and lists a thousand packages one after another. +bash /builder/index-offline-mirror.sh "$offline_mirror_dir" # mkarchiso expects the mirror at /var/cache/omarchy/mirror/offline inside the # container (the airootfs path); symlink rather than duplicate. diff --git a/builder/index-offline-mirror.sh b/builder/index-offline-mirror.sh new file mode 100755 index 0000000..ffa8600 --- /dev/null +++ b/builder/index-offline-mirror.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Index the offline mirror: /offline.db.tar.gz and offline.files.tar.gz +# from every package in , the same databases one repo-add call writes. +# +# index-offline-mirror.sh [jobs] +# +# repo-add handles one package at a time, and for each it hashes the archive +# and lists its files, which means decompressing all of it. A database is only +# a tar of one directory per package, so the packages are dealt out to one +# repo-add per CPU, each writing its own database, and the pieces are put +# together afterwards. +set -euo pipefail + +dir=$(realpath "$1") +jobs=${2:-$(nproc)} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +mapfile -t packages < <(find "$dir" -maxdepth 1 -name '*.pkg.tar.*' ! -name '*.sig' | sort) +(( ${#packages[@]} > 0 )) || { echo "index-offline-mirror: no packages in $dir" >&2; exit 1; } +(( jobs > ${#packages[@]} )) && jobs=${#packages[@]} +(( jobs >= 1 )) || jobs=1 + +# Deal the sorted list out round-robin: big and small packages spread evenly. +for (( i = 0; i < jobs; i++ )); do mkdir "$work/$i"; done +for (( i = 0; i < ${#packages[@]}; i++ )); do + printf '%s\0' "${packages[i]}" >>"$work/$(( i % jobs ))/list" +done + +pids=() +for (( i = 0; i < jobs; i++ )); do + xargs -0 -a "$work/$i/list" repo-add -q "$work/$i/offline.db.tar.gz" >"$work/$i/log" 2>&1 & + pids+=($!) +done +failed=0 +for (( i = 0; i < jobs; i++ )); do + if ! wait "${pids[i]}"; then + failed=1 + echo "index-offline-mirror: repo-add failed on part $i:" >&2 + cat "$work/$i/log" >&2 + fi +done +(( failed == 0 )) || exit 1 + +mkdir "$work/db" "$work/files" +for (( i = 0; i < jobs; i++ )); do + bsdtar -xf "$work/$i/offline.db.tar.gz" -C "$work/db" + bsdtar -xf "$work/$i/offline.files.tar.gz" -C "$work/files" +done + +# One repo-add keeps a single entry per package name. Apart, two versions of +# one name would both survive, so that is an error here rather than a choice. +entries=$(find "$work/db" -mindepth 1 -maxdepth 1 -type d | wc -l) +names=$(cat "$work"/db/*/desc | awk '/^%NAME%$/ { getline; print }' | sort -u | wc -l) +if (( entries != ${#packages[@]} || names != entries )); then + echo "index-offline-mirror: ${#packages[@]} packages gave $entries entries for $names names; two versions of one package?" >&2 + exit 1 +fi + +rm -f "$dir"/offline.db* "$dir"/offline.files* +for kind in db files; do + find "$work/$kind" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | sort | + bsdtar -czf "$dir/offline.$kind.tar.gz" -C "$work/$kind" -T - + ln -s "offline.$kind.tar.gz" "$dir/offline.$kind" +done +echo "Indexed ${#packages[@]} packages with $jobs repo-add processes." diff --git a/test/unit/offline-mirror-index-test.sh b/test/unit/offline-mirror-index-test.sh new file mode 100644 index 0000000..bfc8b5a --- /dev/null +++ b/test/unit/offline-mirror-index-test.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# The offline mirror is indexed by several repo-add processes whose databases +# are then joined (builder/index-offline-mirror.sh). The result has to be the +# database one repo-add writes over the same packages, and two versions of one +# package have to be an error, since apart they would both be kept. + +set -euo pipefail + +ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +index="$ROOT/builder/index-offline-mirror.sh" +bash -n "$index" +if ! command -v repo-add >/dev/null; then + echo "offline mirror index tests skipped: no repo-add here" + exit 0 +fi + +fixture=$(mktemp -d) +trap 'rm -rf "$fixture"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } + +package() { # directory name version: the least repo-add accepts as a package + local stage="$fixture/stage" + rm -rf "$stage"; mkdir -p "$stage/usr/share/$2" + echo "$2 $3" >"$stage/usr/share/$2/version" + printf 'pkgname = %s\npkgver = %s\npkgdesc = fixture\narch = any\nsize = 1\n' "$2" "$3" >"$stage/.PKGINFO" + bsdtar -czf "$1/$2-$3-any.pkg.tar.gz" -C "$stage" .PKGINFO usr +} +contents() { # database file -> directory holding its entries + mkdir -p "$2"; bsdtar -xf "$1" -C "$2" +} + +mkdir "$fixture/one" "$fixture/many" +for name in alpha bravo charlie delta echo foxtrot golf; do + package "$fixture/one" "$name" 1.0-1 + cp "$fixture/one/$name-1.0-1-any.pkg.tar.gz" "$fixture/many/" +done +: >"$fixture/many/alpha-1.0-1-any.pkg.tar.gz.sig" # a signature is not a package + +repo-add -q "$fixture/one/offline.db.tar.gz" "$fixture"/one/*.pkg.tar.gz >/dev/null 2>&1 +bash "$index" "$fixture/many" 3 >/dev/null || fail "indexing seven packages with three processes failed" +for kind in db files; do + [[ -L $fixture/many/offline.$kind ]] || fail "offline.$kind, the name pacman fetches, is missing" + contents "$fixture/one/offline.$kind.tar.gz" "$fixture/x-one-$kind" + contents "$fixture/many/offline.$kind" "$fixture/x-many-$kind" + diff -r "$fixture/x-one-$kind" "$fixture/x-many-$kind" >/dev/null \ + || fail "the joined $kind database differs from one repo-add's" +done + +# More processes than packages, and a second run over an indexed mirror. +bash "$index" "$fixture/many" 64 >/dev/null || fail "more processes than packages failed" +contents "$fixture/many/offline.db" "$fixture/x-again" +diff -r "$fixture/x-one-db" "$fixture/x-again" >/dev/null || fail "indexing an already indexed mirror changed it" + +package "$fixture/many" alpha 2.0-1 +bash "$index" "$fixture/many" 3 >/dev/null 2>&1 && fail "two versions of one package were indexed" + +mkdir "$fixture/empty" +bash "$index" "$fixture/empty" >/dev/null 2>&1 && fail "an empty mirror was indexed" + +grep -Fq 'bash /builder/index-offline-mirror.sh "$offline_mirror_dir"' "$ROOT/builder/build-iso.sh" \ + || fail "the build does not use the parallel index" + +echo "offline mirror index tests passed"