Index the offline mirror with one repo-add per CPU

repo-add takes one package at a time: it hashes the archive and lists its
files, which means decompressing all of it, for about a thousand
packages one after another.

A database is a tar of one directory per package. The packages are dealt
out to one repo-add per CPU, each writing its own database, and the
pieces are joined. On 160 packages on a 16-core machine the result is
identical to one repo-add's, db and files, in 1.0 s against 6.3 s. Two
versions of one package name are an error: one repo-add would keep the
last, the pieces apart would keep both.

builder/build-iso.sh

@@ -304,8 +304,9 @@ printf '%s\n' "${required_package_files[@]}" |
 
 # Rebuild the offline repo db from scratch so size/checksum/depends entries
 # always reflect only the package files selected for this build.
-rm -f "$offline_mirror_dir"/offline.db* "$offline_mirror_dir"/offline.files*
+# Every package archive in the mirror, indexed by one repo-add per CPU: one
-repo-add "$offline_mirror_dir/offline.db.tar.gz" "$offline_mirror_dir/"*.pkg.tar.zst
+# alone hashes and lists a thousand packages one after another.
+bash /builder/index-offline-mirror.sh "$offline_mirror_dir"
 
 # mkarchiso expects the mirror at /var/cache/omarchy/mirror/offline inside the
 # container (the airootfs path); symlink rather than duplicate.

builder/index-offline-mirror.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Index the offline mirror: <dir>/offline.db.tar.gz and offline.files.tar.gz
+# from every package in <dir>, the same databases one repo-add call writes.
+#
+#   index-offline-mirror.sh <dir> [jobs]
+#
+# repo-add handles one package at a time, and for each it hashes the archive
+# and lists its files, which means decompressing all of it. A database is only
+# a tar of one directory per package, so the packages are dealt out to one
+# repo-add per CPU, each writing its own database, and the pieces are put
+# together afterwards.
+set -euo pipefail
+
+dir=$(realpath "$1")
+jobs=${2:-$(nproc)}
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+mapfile -t packages < <(find "$dir" -maxdepth 1 -name '*.pkg.tar.*' ! -name '*.sig' | sort)
+(( ${#packages[@]} > 0 )) || { echo "index-offline-mirror: no packages in $dir" >&2; exit 1; }
+(( jobs > ${#packages[@]} )) && jobs=${#packages[@]}
+(( jobs >= 1 )) || jobs=1
+
+# Deal the sorted list out round-robin: big and small packages spread evenly.
+for (( i = 0; i < jobs; i++ )); do mkdir "$work/$i"; done
+for (( i = 0; i < ${#packages[@]}; i++ )); do
+  printf '%s\0' "${packages[i]}" >>"$work/$(( i % jobs ))/list"
+done
+
+pids=()
+for (( i = 0; i < jobs; i++ )); do
+  xargs -0 -a "$work/$i/list" repo-add -q "$work/$i/offline.db.tar.gz" >"$work/$i/log" 2>&1 &
+  pids+=($!)
+done
+failed=0
+for (( i = 0; i < jobs; i++ )); do
+  if ! wait "${pids[i]}"; then
+    failed=1
+    echo "index-offline-mirror: repo-add failed on part $i:" >&2
+    cat "$work/$i/log" >&2
+  fi
+done
+(( failed == 0 )) || exit 1
+
+mkdir "$work/db" "$work/files"
+for (( i = 0; i < jobs; i++ )); do
+  bsdtar -xf "$work/$i/offline.db.tar.gz" -C "$work/db"
+  bsdtar -xf "$work/$i/offline.files.tar.gz" -C "$work/files"
+done
+
+# One repo-add keeps a single entry per package name. Apart, two versions of
+# one name would both survive, so that is an error here rather than a choice.
+entries=$(find "$work/db" -mindepth 1 -maxdepth 1 -type d | wc -l)
+names=$(cat "$work"/db/*/desc | awk '/^%NAME%$/ { getline; print }' | sort -u | wc -l)
+if (( entries != ${#packages[@]} || names != entries )); then
+  echo "index-offline-mirror: ${#packages[@]} packages gave $entries entries for $names names; two versions of one package?" >&2
+  exit 1
+fi
+
+rm -f "$dir"/offline.db* "$dir"/offline.files*
+for kind in db files; do
+  find "$work/$kind" -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | sort |
+    bsdtar -czf "$dir/offline.$kind.tar.gz" -C "$work/$kind" -T -
+  ln -s "offline.$kind.tar.gz" "$dir/offline.$kind"
+done
+echo "Indexed ${#packages[@]} packages with $jobs repo-add processes."

test/unit/offline-mirror-index-test.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# The offline mirror is indexed by several repo-add processes whose databases
+# are then joined (builder/index-offline-mirror.sh). The result has to be the
+# database one repo-add writes over the same packages, and two versions of one
+# package have to be an error, since apart they would both be kept.
+
+set -euo pipefail
+
+ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)
+index="$ROOT/builder/index-offline-mirror.sh"
+bash -n "$index"
+if ! command -v repo-add >/dev/null; then
+  echo "offline mirror index tests skipped: no repo-add here"
+  exit 0
+fi
+
+fixture=$(mktemp -d)
+trap 'rm -rf "$fixture"' EXIT
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+package() { # directory name version: the least repo-add accepts as a package
+  local stage="$fixture/stage"
+  rm -rf "$stage"; mkdir -p "$stage/usr/share/$2"
+  echo "$2 $3" >"$stage/usr/share/$2/version"
+  printf 'pkgname = %s\npkgver = %s\npkgdesc = fixture\narch = any\nsize = 1\n' "$2" "$3" >"$stage/.PKGINFO"
+  bsdtar -czf "$1/$2-$3-any.pkg.tar.gz" -C "$stage" .PKGINFO usr
+}
+contents() { # database file -> directory holding its entries
+  mkdir -p "$2"; bsdtar -xf "$1" -C "$2"
+}
+
+mkdir "$fixture/one" "$fixture/many"
+for name in alpha bravo charlie delta echo foxtrot golf; do
+  package "$fixture/one" "$name" 1.0-1
+  cp "$fixture/one/$name-1.0-1-any.pkg.tar.gz" "$fixture/many/"
+done
+: >"$fixture/many/alpha-1.0-1-any.pkg.tar.gz.sig"   # a signature is not a package
+
+repo-add -q "$fixture/one/offline.db.tar.gz" "$fixture"/one/*.pkg.tar.gz >/dev/null 2>&1
+bash "$index" "$fixture/many" 3 >/dev/null || fail "indexing seven packages with three processes failed"
+for kind in db files; do
+  [[ -L $fixture/many/offline.$kind ]] || fail "offline.$kind, the name pacman fetches, is missing"
+  contents "$fixture/one/offline.$kind.tar.gz" "$fixture/x-one-$kind"
+  contents "$fixture/many/offline.$kind" "$fixture/x-many-$kind"
+  diff -r "$fixture/x-one-$kind" "$fixture/x-many-$kind" >/dev/null \
+    || fail "the joined $kind database differs from one repo-add's"
+done
+
+# More processes than packages, and a second run over an indexed mirror.
+bash "$index" "$fixture/many" 64 >/dev/null || fail "more processes than packages failed"
+contents "$fixture/many/offline.db" "$fixture/x-again"
+diff -r "$fixture/x-one-db" "$fixture/x-again" >/dev/null || fail "indexing an already indexed mirror changed it"
+
+package "$fixture/many" alpha 2.0-1
+bash "$index" "$fixture/many" 3 >/dev/null 2>&1 && fail "two versions of one package were indexed"
+
+mkdir "$fixture/empty"
+bash "$index" "$fixture/empty" >/dev/null 2>&1 && fail "an empty mirror was indexed"
+
+grep -Fq 'bash /builder/index-offline-mirror.sh "$offline_mirror_dir"' "$ROOT/builder/build-iso.sh" \
+  || fail "the build does not use the parallel index"
+
+echo "offline mirror index tests passed"