diff --git a/docs/file-layout.md b/docs/file-layout.md index d612166a..09065807 100644 --- a/docs/file-layout.md +++ b/docs/file-layout.md @@ -256,7 +256,8 @@ the legacy finalization marker from `~/.local/state/omarchy/` into `done/`. finalization. It sources: - `install/config/all.sh` — theme links, lockout limits, lockscreen PAM, - powerprofilesctl shebang fix, docker setup, service enablement, firewall. + powerprofilesctl shebang fix, docker setup, Snapper retention, locate + index tuning, service enablement, firewall. - `install/hardware/all.sh` via `omarchy-setup-hardware` — vendor- and device-specific kernel modules, udev rules, microcode, wireless regdom, ASUS / Framework / Intel / Apple / Lenovo quirks. diff --git a/install/config/all.sh b/install/config/all.sh index e416b7bc..c4c70752 100644 --- a/install/config/all.sh +++ b/install/config/all.sh @@ -4,5 +4,6 @@ run_logged "$OMARCHY_INSTALL/config/lockscreen-pam.sh" run_logged "$OMARCHY_INSTALL/config/fix-powerprofilesctl-shebang.sh" run_logged "$OMARCHY_INSTALL/config/docker.sh" run_logged "$OMARCHY_INSTALL/config/snapper.sh" +run_logged "$OMARCHY_INSTALL/config/locate.sh" run_logged "$OMARCHY_INSTALL/config/enable-services.sh" run_logged "$OMARCHY_INSTALL/config/firewall.sh" diff --git a/install/config/locate.sh b/install/config/locate.sh new file mode 100644 index 00000000..cc95e9b8 --- /dev/null +++ b/install/config/locate.sh @@ -0,0 +1,32 @@ +UPDATEDB_CONF_PATH="${OMARCHY_UPDATEDB_CONF_PATH:-/etc/updatedb.conf}" + +echo "Configuring locate to skip Btrfs snapshots and index Btrfs subvolumes" + +[[ -f $UPDATEDB_CONF_PATH ]] || exit 0 + +# updatedb refuses to run at all on a config that defines a variable twice, so +# every setting here is rewritten where it already stands and only appended +# when the file has no line for it. + +# Btrfs subvolume mounts (like /home) look like bind mounts, so pruning +# bind mounts leaves them out of the index entirely. +if grep -qE '^[[:space:]]*PRUNE_BIND_MOUNTS[[:space:]]*=' "$UPDATEDB_CONF_PATH"; then + sed -i -E 's|^[[:space:]]*PRUNE_BIND_MOUNTS[[:space:]]*=.*|PRUNE_BIND_MOUNTS = "no"|' "$UPDATEDB_CONF_PATH" +else + printf '%s\n' 'PRUNE_BIND_MOUNTS = "no"' >>"$UPDATEDB_CONF_PATH" +fi + +# Snapper snapshots are nested subvolumes reached by plain directory +# traversal, so without this updatedb indexes the system once per snapshot. +if grep -qE '^[[:space:]]*PRUNEPATHS[[:space:]]*=' "$UPDATEDB_CONF_PATH"; then + # updatedb only accepts quoted values and allows a comment after them. Read + # back what the machine already prunes and write the whole setting out again + # rather than splicing into a line of unknown shape. + pruned=$(sed -nE 's|^[[:space:]]*PRUNEPATHS[[:space:]]*=[[:space:]]*"([^"]*)".*|\1|p' "$UPDATEDB_CONF_PATH" | tail -n 1) + + if [[ " $pruned " != *" /.snapshots "* ]]; then + sed -i -E "s|^[[:space:]]*PRUNEPATHS[[:space:]]*=.*|PRUNEPATHS = \"/.snapshots${pruned:+ $pruned}\"|" "$UPDATEDB_CONF_PATH" + fi +else + printf '%s\n' 'PRUNEPATHS = "/.snapshots"' >>"$UPDATEDB_CONF_PATH" +fi diff --git a/migrations/1784809451.sh b/migrations/1784809451.sh new file mode 100644 index 00000000..fb59a794 --- /dev/null +++ b/migrations/1784809451.sh @@ -0,0 +1,30 @@ +echo "Configure locate to skip Btrfs snapshots and index Btrfs subvolumes" + +OMARCHY_PATH="${OMARCHY_PATH:-/usr/share/omarchy}" +locate_config_script="$OMARCHY_PATH/install/config/locate.sh" +UPDATEDB_CONF_PATH="${OMARCHY_UPDATEDB_CONF_PATH:-/etc/updatedb.conf}" + +as_root() { + if (( EUID == 0 )); then + "$@" + else + sudo "$@" + fi +} + +[[ -f $UPDATEDB_CONF_PATH ]] || exit 0 +[[ -f $locate_config_script ]] || exit 0 + +if grep -q '^PRUNE_BIND_MOUNTS = "no"' "$UPDATEDB_CONF_PATH" && + grep -E '^PRUNEPATHS' "$UPDATEDB_CONF_PATH" | grep -qE '(^|[[:space:]"])/\.snapshots([[:space:]"]|$)'; then + exit 0 +fi + +as_root env OMARCHY_UPDATEDB_CONF_PATH="$UPDATEDB_CONF_PATH" bash -euo pipefail "$locate_config_script" + +# Rebuild the index with the new exclusions; pruning /.snapshots turns +# multi-hour runs on snapshot-heavy systems back into one-minute runs. Restart +# rather than start: the machines this targets are the ones with an updatedb +# already grinding through every snapshot, and a run that started before the +# rewrite keeps using the config it read at startup. +as_root systemctl restart --no-block plocate-updatedb.service >/dev/null 2>&1 || true diff --git a/migrations/1784809452.sh b/migrations/1784809452.sh new file mode 100644 index 00000000..d70d7659 --- /dev/null +++ b/migrations/1784809452.sh @@ -0,0 +1,56 @@ +echo "Remove Snapper timeline snapshots leaked by earlier defaults" + +SNAPPER_CONFIG_PATH="${OMARCHY_SNAPPER_CONFIG_PATH:-/etc/snapper/configs/root}" + +as_root() { + if (( EUID == 0 )); then + "$@" + else + sudo "$@" + fi +} + +command -v snapper >/dev/null || exit 0 +[[ -f $SNAPPER_CONFIG_PATH ]] || exit 0 + +# Only clean up when timeline snapshotting is off, as Omarchy configures it. +# Anyone who deliberately turned it back on keeps their snapshots. Snapper's +# own create-config leaves the file readable by root alone, and a config this +# user cannot read must not pass for one that wants its snapshots kept. +if [[ -r $SNAPPER_CONFIG_PATH ]]; then + grep -qFx 'TIMELINE_CREATE="no"' "$SNAPPER_CONFIG_PATH" || exit 0 +else + as_root grep -qFx 'TIMELINE_CREATE="no"' "$SNAPPER_CONFIG_PATH" || exit 0 +fi + +# Earlier installs ran hourly timeline snapshots. Later configs stopped +# creating them but never deleted the existing ones, and number cleanup +# skips snapshots marked Cleanup=timeline, so they pile up forever: +# hundreds of snapshots pinning 100+ GB of extents on long-running machines. +leaked=$(as_root snapper -c root --csvout list --columns number,cleanup 2>/dev/null | awk -F, '$2 == "timeline" { print $1 }' || true) +[[ -n $leaked ]] || exit 0 + +echo "Deleting $(wc -w <<<"$leaked") leaked timeline snapshots (disk space is reclaimed in the background)" + +# Delete in small batches; one big delete can die on a DBus timeout partway. +# A failed batch must not take the rest of the migration run down with it, so +# the drain is best effort. omarchy-migrate records the migration either way, +# so say what is left rather than counting on a rerun that will not come. +failed=0 +batch=() + +for number in $leaked; do + batch+=("$number") + if (( ${#batch[@]} == 20 )); then + as_root snapper -c root delete "${batch[@]}" || failed=$((failed + ${#batch[@]})) + batch=() + fi +done + +if (( ${#batch[@]} > 0 )); then + as_root snapper -c root delete "${batch[@]}" || failed=$((failed + ${#batch[@]})) +fi + +if (( failed > 0 )); then + echo "$failed snapshots could not be deleted. Finish with: sudo snapper -c root delete " +fi diff --git a/test/shell.d/locate-test.sh b/test/shell.d/locate-test.sh new file mode 100644 index 00000000..cab54c24 --- /dev/null +++ b/test/shell.d/locate-test.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +set -euo pipefail + +source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh" + +config_script="$ROOT/install/config/locate.sh" + +test_tmp=$(mktemp -d) +trap 'rm -rf "$test_tmp"' EXIT + +stock_conf() { + cat >"$1" <<'CONF' +PRUNE_BIND_MOUNTS = "yes" +PRUNEFS = "9p afs autofs cifs fuse nfs nfs4 proc sysfs tmpfs" +PRUNENAMES = ".git .hg .svn" +PRUNEPATHS = "/afs /media /mnt /net /sfs /tmp /udev /var/cache /var/lib/pacman/local /var/lock /var/run /var/spool /var/tmp" +CONF +} + +# updatedb dies on a config that defines a variable twice, so hand every +# rewritten file to the real parser rather than trusting the greps below. +empty_tree="$test_tmp/empty-tree" +mkdir -p "$empty_tree" + +assert_conf_parses() { + command -v updatedb >/dev/null || return 0 + + local errors + errors=$(updatedb --config-file "$1" -U "$empty_tree" -o "$test_tmp/plocate.db" 2>&1 >/dev/null | grep -F "$1:" || true) + [[ -z $errors ]] || fail "updatedb accepts the rewritten config" "$errors" +} + +conf="$test_tmp/updatedb.conf" +stock_conf "$conf" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +grep -qFx 'PRUNE_BIND_MOUNTS = "no"' "$conf" || fail "locate config indexes Btrfs subvolume mounts like /home" +grep -qF 'PRUNEPATHS = "/.snapshots /afs' "$conf" || fail "locate config prunes /.snapshots" +assert_conf_parses "$conf" +pass "locate config skips Btrfs snapshots and indexes Btrfs subvolumes" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +[[ $(grep -o '/\.snapshots' "$conf" | wc -l) -eq 1 ]] || fail "locate config is idempotent" +assert_conf_parses "$conf" +pass "locate config leaves an already-configured file alone" + +OMARCHY_UPDATEDB_CONF_PATH="$test_tmp/missing.conf" bash -euo pipefail "$config_script" >/dev/null +pass "locate config tolerates a missing updatedb.conf" + +# A hand-edited updatedb.conf may drop the settings entirely, or write them +# without the spaces around the "=" or the quotes that the stock Arch file uses. +conf="$test_tmp/sparse-updatedb.conf" +printf '%s\n' 'PRUNENAMES = ".git .hg .svn"' >"$conf" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +grep -qFx 'PRUNE_BIND_MOUNTS = "no"' "$conf" || fail "locate config adds a missing PRUNE_BIND_MOUNTS" +grep -qFx 'PRUNEPATHS = "/.snapshots"' "$conf" || fail "locate config adds a missing PRUNEPATHS" +assert_conf_parses "$conf" +pass "locate config adds settings a hand-edited updatedb.conf is missing" + +conf="$test_tmp/unspaced-updatedb.conf" +printf '%s\n' 'PRUNE_BIND_MOUNTS="yes"' 'PRUNEPATHS="/tmp /var/tmp"' >"$conf" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +grep -qFx 'PRUNE_BIND_MOUNTS = "no"' "$conf" || fail "locate config rewrites an unspaced PRUNE_BIND_MOUNTS" +grep -qFx 'PRUNEPATHS = "/.snapshots /tmp /var/tmp"' "$conf" || fail "locate config prunes /.snapshots in an unspaced PRUNEPATHS" +[[ $(grep -c 'PRUNEPATHS' "$conf") -eq 1 ]] || fail "locate config keeps a single PRUNEPATHS setting" +assert_conf_parses "$conf" +pass "locate config handles updatedb.conf written without spaces around =" + +# updatedb allows a comment after a value and indented settings, and defining +# either setting twice makes it refuse to run at all. +conf="$test_tmp/commented-updatedb.conf" +printf '%s\n' ' PRUNE_BIND_MOUNTS = "yes" # subvolumes look like bind mounts' \ + 'PRUNEPATHS = "/tmp" # scratch' >"$conf" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +grep -qFx 'PRUNE_BIND_MOUNTS = "no"' "$conf" || fail "locate config rewrites an indented PRUNE_BIND_MOUNTS" +grep -qFx 'PRUNEPATHS = "/.snapshots /tmp"' "$conf" || fail "locate config keeps the paths a commented PRUNEPATHS already prunes" +[[ $(grep -c 'PRUNEPATHS' "$conf") -eq 1 ]] || fail "locate config replaces a commented PRUNEPATHS instead of adding a second one" +assert_conf_parses "$conf" +pass "locate config handles indented settings and trailing comments" + +# A hand-edited file may have dropped the quotes updatedb requires, which +# leaves it unparseable until something writes the setting out properly. +conf="$test_tmp/unquoted-updatedb.conf" +printf '%s\n' 'PRUNEPATHS = /tmp' >"$conf" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +grep -qFx 'PRUNEPATHS = "/.snapshots"' "$conf" || fail "locate config repairs an unquoted PRUNEPATHS" +[[ $(grep -c 'PRUNEPATHS' "$conf") -eq 1 ]] || fail "locate config replaces an unquoted PRUNEPATHS instead of adding a second one" +assert_conf_parses "$conf" +pass "locate config handles updatedb.conf written without quotes" + +# A path that merely ends in /.snapshots is not the root snapshot directory. +conf="$test_tmp/nested-snapshots-updatedb.conf" +printf '%s\n' 'PRUNEPATHS = "/var/lib/machines/.snapshots"' >"$conf" + +OMARCHY_UPDATEDB_CONF_PATH="$conf" bash -euo pipefail "$config_script" >/dev/null + +grep -qFx 'PRUNEPATHS = "/.snapshots /var/lib/machines/.snapshots"' "$conf" || fail "locate config prunes /.snapshots alongside a path that ends in it" +assert_conf_parses "$conf" +pass "locate config tells /.snapshots apart from a path that ends in it" + +locate_migration=$(grep -rl 'Configure locate to skip Btrfs snapshots' "$ROOT/migrations" | head -n 1 || true) +[[ -n $locate_migration ]] || fail "locate migration exists" + +fake_bin="$test_tmp/bin" +mkdir -p "$fake_bin" + +cat >"$fake_bin/sudo" <<'STUB' +#!/bin/bash +exec "$@" +STUB +chmod +x "$fake_bin/sudo" + +cat >"$fake_bin/systemctl" <<'STUB' +#!/bin/bash +printf 'systemctl %s\n' "$*" >>"$TEST_LOG" +STUB +chmod +x "$fake_bin/systemctl" + +conf="$test_tmp/migration-updatedb.conf" +stock_conf "$conf" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_PATH="$ROOT" \ +OMARCHY_UPDATEDB_CONF_PATH="$conf" \ + bash -euo pipefail "$locate_migration" >/dev/null + +grep -qFx 'PRUNE_BIND_MOUNTS = "no"' "$conf" || fail "locate migration rewrites updatedb.conf" +grep -qF 'PRUNEPATHS = "/.snapshots /afs' "$conf" || fail "locate migration prunes /.snapshots" +grep -qFx 'systemctl restart --no-block plocate-updatedb.service' "$test_tmp/calls.log" || fail "locate migration replaces an in-flight run and rebuilds the index without blocking" +pass "locate migration fixes existing installs and rebuilds the index" + +: >"$test_tmp/calls.log" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_PATH="$ROOT" \ +OMARCHY_UPDATEDB_CONF_PATH="$conf" \ + bash -euo pipefail "$locate_migration" >/dev/null + +[[ ! -s $test_tmp/calls.log ]] || fail "locate migration skips already-configured installs" +pass "locate migration is a no-op once updatedb.conf is configured" + +# A dev checkout carries migrations from a release whose install scripts the +# checked-out tree may not have yet, and omarchy-migrate runs under set -e. +: >"$test_tmp/calls.log" +conf="$test_tmp/no-config-script-updatedb.conf" +stock_conf "$conf" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_PATH="$test_tmp/empty" \ +OMARCHY_UPDATEDB_CONF_PATH="$conf" \ + bash -euo pipefail "$locate_migration" >/dev/null || + fail "locate migration survives a tree without the locate config script" + +[[ ! -s $test_tmp/calls.log ]] || fail "locate migration touches nothing without the locate config script" +pass "locate migration is a no-op when the locate config script is missing" diff --git a/test/shell.d/snapper-timeline-leak-test.sh b/test/shell.d/snapper-timeline-leak-test.sh new file mode 100644 index 00000000..21b15ac7 --- /dev/null +++ b/test/shell.d/snapper-timeline-leak-test.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +set -euo pipefail + +source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh" + +leak_migration=$(grep -rl 'timeline snapshots leaked by earlier defaults' "$ROOT/migrations" | head -n 1 || true) +[[ -n $leak_migration ]] || fail "Snapper timeline leak migration exists" + +test_tmp=$(mktemp -d) +trap 'rm -rf "$test_tmp"' EXIT + +fake_bin="$test_tmp/bin" +mkdir -p "$fake_bin" + +cat >"$fake_bin/sudo" <<'STUB' +#!/bin/bash +printf 'sudo %s\n' "$*" >>"$TEST_LOG" +exec "$@" +STUB +chmod +x "$fake_bin/sudo" + +cat >"$fake_bin/snapper" <<'STUB' +#!/bin/bash +printf 'snapper %s\n' "$*" >>"$TEST_LOG" +if [[ "$*" == *"--csvout list"* ]]; then + echo "number,cleanup" + for i in $(seq 1 45); do + echo "$i,timeline" + done + echo "100,number" + echo "101," +fi +STUB +chmod +x "$fake_bin/snapper" + +snapper_config="$test_tmp/root" +printf '%s\n' 'TIMELINE_CREATE="no"' 'NUMBER_CLEANUP="yes"' >"$snapper_config" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_SNAPPER_CONFIG_PATH="$snapper_config" \ + bash -euo pipefail "$leak_migration" >/dev/null + +deletes=$(grep -c '^snapper -c root delete ' "$test_tmp/calls.log" || true) +[[ $deletes -eq 3 ]] || fail "leak migration deletes snapshots in batches" "expected 3 delete calls, got $deletes" + +first_batch=$(grep -m1 '^snapper -c root delete ' "$test_tmp/calls.log") +[[ $first_batch == "snapper -c root delete $(seq -s ' ' 1 20)" ]] || fail "leak migration caps delete batches at 20 snapshots" "$first_batch" + +last_batch=$(grep '^snapper -c root delete ' "$test_tmp/calls.log" | tail -n 1) +[[ $last_batch == "snapper -c root delete $(seq -s ' ' 41 45)" ]] || fail "leak migration deletes the final partial batch" "$last_batch" + +! grep -E '^snapper -c root delete .*\b(100|101)\b' "$test_tmp/calls.log" || fail "leak migration only deletes timeline snapshots" +pass "leak migration removes leaked timeline snapshots in batches and keeps the rest" + +# omarchy-migrate runs under set -e, so a batch that dies on a DBus timeout +# would otherwise abort the run and skip every migration queued behind it. +: >"$test_tmp/calls.log" +printf '%s\n' 'TIMELINE_CREATE="no"' 'NUMBER_CLEANUP="yes"' >"$snapper_config" + +cat >"$fake_bin/snapper" <<'STUB' +#!/bin/bash +printf 'snapper %s\n' "$*" >>"$TEST_LOG" +if [[ "$*" == *"--csvout list"* ]]; then + echo "number,cleanup" + for i in $(seq 1 45); do + echo "$i,timeline" + done + exit 0 +fi +echo "failure: dbus timeout" >&2 +exit 1 +STUB + +output=$(TEST_LOG="$test_tmp/calls.log" \ + PATH="$fake_bin:$PATH" \ + OMARCHY_SNAPPER_CONFIG_PATH="$snapper_config" \ + bash -euo pipefail "$leak_migration" 2>/dev/null) || + fail "leak migration survives a failed delete batch" + +deletes=$(grep -c '^snapper -c root delete ' "$test_tmp/calls.log" || true) +[[ $deletes -eq 3 ]] || fail "leak migration keeps draining after a failed batch" "expected 3 delete calls, got $deletes" + +# omarchy-migrate writes the completion marker even when the drain gave up, so +# what is left has to be said out loud rather than left for a rerun. +grep -qF '45 snapshots could not be deleted' <<<"$output" || fail "leak migration reports the snapshots it could not delete" "$output" +pass "leak migration tolerates a batch that fails partway" + +: >"$test_tmp/calls.log" +printf '%s\n' 'TIMELINE_CREATE="yes"' >"$snapper_config" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_SNAPPER_CONFIG_PATH="$snapper_config" \ + bash -euo pipefail "$leak_migration" >/dev/null + +[[ ! -s $test_tmp/calls.log ]] || fail "leak migration leaves deliberate timeline setups alone" +pass "leak migration skips systems where timeline snapshots are intentional" + +: >"$test_tmp/calls.log" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_SNAPPER_CONFIG_PATH="$test_tmp/missing" \ + bash -euo pipefail "$leak_migration" >/dev/null + +[[ ! -s $test_tmp/calls.log ]] || fail "leak migration skips systems without a Snapper root config" +pass "leak migration is a no-op without Snapper configured" + +# Snapper's create-config writes a root-only config, and a config this user +# cannot read says nothing about whether timeline snapshots are wanted. +: >"$test_tmp/calls.log" +printf '%s\n' 'TIMELINE_CREATE="no"' >"$snapper_config" +chmod 000 "$snapper_config" + +TEST_LOG="$test_tmp/calls.log" \ +PATH="$fake_bin:$PATH" \ +OMARCHY_SNAPPER_CONFIG_PATH="$snapper_config" \ + bash -euo pipefail "$leak_migration" >/dev/null 2>&1 + +chmod 600 "$snapper_config" + +grep -qF "sudo grep -qFx TIMELINE_CREATE=\"no\" $snapper_config" "$test_tmp/calls.log" || + fail "leak migration reads a root-only Snapper config as root" "$(cat "$test_tmp/calls.log")" +pass "leak migration does not mistake an unreadable Snapper config for an intentional one"