Pin root= before the packages that can drop it (#6951)

* Pin root= before the packages that can drop it

limine-entry-tool falls back to /proc/cmdline for root= only while nothing
appends to KERNEL_CMDLINE. Installing omarchy-settings lands a drop-in that
appends with +=, switching that fallback off, and a kernel bump in the same
transaction then bakes a UKI with no root= at all. preserve_kernel_cmdline_root
repaired that afterwards, so a completed upgrade booted — but the machine was
unbootable for the seconds in between, and an upgrade interrupted there left it
in an emergency shell.

Pinning cannot wait until after the packages land, and the old guard could not
be moved earlier as it was: it asked the tool for its effective cmdline, which
still resolves root= through the fallback right up until the drop-in arrives,
so it reported healthy on exactly the machines about to break. Ask the config
layers whether root= is stated explicitly instead, for the default profile
alone, and pin before the package transaction. Verification stays after it,
since that is what rebuilds the UKIs.

The pin no longer gates on limine-mkinitcpio being present, since it now runs
before the transaction that can install it, and a machine still missing it is
exactly one that needs pinning first.

Verified in a VM upgrading a 3.8.0 install, sampling the UKI every 2s across the
upgrade: a legacy install without the pin lost root= for ~10s, and booting that
state landed in "Failed to mount '' on real root". With this change the same
upgrade never loses it, and an install that already pins root= is untouched.

Closes #6894

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* Merge the limine config layers when checking for a root= pin, and re-pin after the transaction

The pin check returned on any line that mentioned root= under KERNEL_CMDLINE[default]. limine-entry-tool merges its layers in order, /usr/share drop-ins, /etc/limine-entry-tool.conf, /etc drop-ins, then /etc/default/limine, where = replaces and += appends, so a later layer's = could remove a pin the check still counted, and a quote before root= let systemd.setenv="root=..." read as one. Both skip the pin, which is the direction that bricks the next boot. The check now merges the layers the same way, strips only the outer double quotes the tool strips, drops quoted values the kernel does not split, and looks for root= as a parameter of its own. It was checked against limine-entry-tool --get-cmdline on a worker for every fixture in the test.

Installing limine-mkinitcpio-hook runs limine-install, so a machine with no /boot/limine.conf when the pin runs ahead of the transaction can have one after it, and verification only warned about the UKI it found without root=. Verification now runs the pin for any machine the first call skipped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex Medium <noreply@openai.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: omarchybot <omarchybot@users.noreply.github.com>
Co-authored-by: Codex Medium <noreply@openai.com>

bin/omarchy-upgrade-to-quattro

@@ -74,6 +74,7 @@ target_user="${OMARCHY_INSTALL_USER:-}"
 yes=0
 auto_reboot=0
 boot_cmdline_unsafe=0
+kernel_cmdline_root_checked=0
 use_dev_packages=${OMARCHY_UPGRADE_DEV:-0}
 
 while (($#)); do
@@ -520,27 +521,62 @@ normalize_limine_config() {
   fi
 }
 
+# True when the limine config layers state root= themselves, rather than leaving
+# it to the /proc/cmdline fallback that the first += drop-in switches off.
+# Only the default profile matters here: that is the key this function appends
+# to and the one the boot entries are built from. The layers are merged in
+# limine-entry-tool's order, where = replaces and += appends, so a later layer
+# can take away a root= an earlier one stated. Globbing and reading both run as
+# root so an unreadable drop-in directory cannot read as "nothing is pinned".
+kernel_cmdline_root_pinned() {
+  as_root bash -c '
+    shopt -s nullglob extglob
+    cmdline=""
+    for conf in /usr/share/limine-entry-tool.d/*.conf /etc/limine-entry-tool.conf /etc/limine-entry-tool.d/*.conf /etc/default/limine; do
+      [[ -f $conf ]] || continue
+      while IFS= read -r line || [[ -n $line ]]; do
+        [[ $line =~ ^[[:space:]]*KERNEL_CMDLINE\[default\][[:space:]]*(\+?)=[[:space:]]*(.*)$ ]] || continue
+        value=${BASH_REMATCH[2]}
+        [[ $value == \"*\" ]] && value=${value:1:-1}
+        if [[ -n ${BASH_REMATCH[1]} ]]; then
+          cmdline+=" $value"
+        else
+          cmdline=$value
+        fi
+      done <"$conf"
+    done
+    # The kernel splits parameters only outside double quotes, and an unmatched
+    # quote runs to the end of the line.
+    cmdline=${cmdline//\"*([^\"])\"/_}
+    [[ " ${cmdline%%\"*}" =~ [[:space:]]root= ]]
+  '
+}
+
 preserve_kernel_cmdline_root() {
   local default_conf=/etc/default/limine
-  local cmdline param key root_source root_stack root_uuid subvol uki
+  local cmdline param key root_source root_stack root_uuid subvol
   local boot_params=()
-  local missing=()
   local have_root=0 have_mount_mode=0 have_unlock=0
 
-  command -v limine-mkinitcpio >/dev/null 2>&1 || return 0
+  # Gate on limine being what boots this machine, not on limine-mkinitcpio being
+  # installed: this now runs before the transaction that can install it, and a
+  # machine still missing it is exactly one that must be pinned first.
   as_root test -f /boot/limine.conf || return 0
+  kernel_cmdline_root_checked=1
 
   # The omarchy-defaults.conf drop-in appends to KERNEL_CMDLINE[default] with
   # +=, which makes limine-entry-tool ignore /etc/kernel/cmdline and
   # /proc/cmdline. Fresh installs pin root= in /etc/default/limine via the ISO;
   # upgrades never created that file, so root= silently drops out and the next
-  # boot lands in an emergency shell. Ask the tool for its effective default
-  # cmdline — the key this function appends to — rather than parsing the config
-  # layers ourselves.
-  if as_root limine-entry-tool --get-cmdline default 2>/dev/null |
-    grep -qE '(^|[[:space:]])root='; then
-    return 0
-  fi
+  # boot lands in an emergency shell.
+  #
+  # Ask the config layers whether root= is stated explicitly rather than asking
+  # the tool for its effective cmdline. The effective cmdline still resolves
+  # root= from the /proc/cmdline fallback right up until the first += drop-in
+  # lands, so it reports healthy on exactly the machines that are about to
+  # break — and this has to run before the package transaction installs that
+  # drop-in.
+  kernel_cmdline_root_pinned && return 0
 
   # Copy the boot-critical parameters of the running kernel verbatim, so a
   # PARTUUID, a LABEL or a bare device node all survive.
@@ -610,6 +646,21 @@ preserve_kernel_cmdline_root() {
 # root filesystem have to be stated explicitly.
 KERNEL_CMDLINE[default]+=" ${boot_params[*]}"
 EOF
+}
+
+# Runs after the package transaction, which is what rebuilds the UKIs through
+# the limine hook. Regenerating here too keeps a machine whose kernel was not
+# bumped from carrying stale entries.
+verify_kernel_cmdline_root() {
+  local uki
+  local missing=()
+
+  command -v limine-mkinitcpio >/dev/null 2>&1 || return 0
+  as_root test -f /boot/limine.conf || return 0
+
+  # Installing limine-mkinitcpio-hook deploys limine, so a machine can have a
+  # limine.conf now that had none when the pin ran ahead of the transaction.
+  ((kernel_cmdline_root_checked)) || preserve_kernel_cmdline_root
 
   # Keep going on failure so the verification below still runs; the unsafe
   # flag blocks the reboot at the end of the upgrade.
@@ -2360,10 +2411,11 @@ install_keyrings
 remove_legacy_installer_package
 remove_legacy_limine_configs
 remove_conflicting_legacy_packages
+preserve_kernel_cmdline_root
 install_omarchy_quattro_packages
 install_hardware_transition_packages
 normalize_limine_config
-preserve_kernel_cmdline_root
+verify_kernel_cmdline_root
 configure_snapper_policy
 configure_lock_authentication
 migrate_1password_beta_package

test/shell.d/upgrade-to-quattro-test.sh

@@ -250,8 +250,14 @@ pass "Omarchy 4 upgrade removes stale nofile drop-ins"
 
 cmdline_line=$(grep -n '^preserve_kernel_cmdline_root$' "$upgrade_to_quattro" | cut -d: -f1)
 packages_line=$(grep -n '^install_omarchy_quattro_packages$' "$upgrade_to_quattro" | cut -d: -f1)
-[[ -n $cmdline_line && -n $packages_line ]] || fail "kernel cmdline preservation and package install calls exist"
-(( packages_line < cmdline_line )) || fail "kernel cmdline preservation runs once limine-mkinitcpio is installed"
+verify_line=$(grep -n '^verify_kernel_cmdline_root$' "$upgrade_to_quattro" | cut -d: -f1)
+[[ -n $cmdline_line && -n $packages_line && -n $verify_line ]] ||
+  fail "kernel cmdline preservation, verification and package install calls exist"
+# The package transaction installs the += drop-in that drops root=, so the pin
+# has to be on disk before it runs or the UKI it bakes is unbootable.
+(( cmdline_line < packages_line )) || fail "kernel cmdline is pinned before the packages that can drop root="
+# The UKIs are rebuilt by the transaction, so they can only be checked after it.
+(( verify_line > packages_line )) || fail "kernel cmdline is verified after the packages are installed"
 grep -F '/etc/default/limine' "$upgrade_to_quattro" >/dev/null
 grep -F 'KERNEL_CMDLINE[default]+=" ${boot_params[*]}"' "$upgrade_to_quattro" >/dev/null
 grep -F 'cat /proc/cmdline' "$upgrade_to_quattro" >/dev/null
@@ -260,13 +266,84 @@ grep -F 'rootflags=subvol=' "$upgrade_to_quattro" >/dev/null
 grep -F 'cryptdevice' "$upgrade_to_quattro" >/dev/null
 pass "Omarchy 4 upgrade preserves the kernel cmdline root parameters"
 
-# The += drop-ins make limine-entry-tool ignore /etc/kernel/cmdline and
-# /proc/cmdline, so only the tool's own merge can say whether root= survives.
-# Queried for the default key, so a kernel-specific pin cannot cover for the
-# entries this repairs.
-grep -F 'limine-entry-tool --get-cmdline default' "$upgrade_to_quattro" >/dev/null
-grep -F "grep -qE '(^|[[:space:]])root='" "$upgrade_to_quattro" >/dev/null
-pass "Omarchy 4 upgrade asks limine-entry-tool whether root= survives"
+# The tool's effective cmdline still resolves root= from /proc/cmdline until the
+# first += drop-in lands, so it reads healthy on exactly the machines about to
+# break. Ask the config layers whether root= is stated instead, for the default
+# key alone so a fallback or kernel-specific pin cannot cover for it.
+grep -F 'kernel_cmdline_root_pinned' "$upgrade_to_quattro" >/dev/null
+grep -F 'KERNEL_CMDLINE\[default\]' "$upgrade_to_quattro" >/dev/null
+! grep -F 'limine-entry-tool --get-cmdline' "$upgrade_to_quattro" >/dev/null ||
+  fail "the pin check does not depend on the fallback it is about to lose"
+pass "Omarchy 4 upgrade checks whether root= is pinned in the limine config"
+
+# Run the pin check against fixture config layers. A false positive skips the
+# pin and the next boot has no root=; a false negative appends a second pin.
+eval "$(sed -n '/^kernel_cmdline_root_pinned() {$/,/^}$/p' "$upgrade_to_quattro")"
+pin_root=$(mktemp -d)
+trap 'rm -rf "$pin_root"' EXIT
+as_root() {
+  local script=${3//\/etc\//$pin_root/etc/}
+  bash -c "${script//\/usr\/share\//$pin_root/usr/share/}"
+}
+# Takes pairs of a config layer and a line to append to it.
+root_pinned() {
+  rm -rf "${pin_root:?}"/{etc,usr}
+  mkdir -p "$pin_root/etc/default" "$pin_root/etc/limine-entry-tool.d" "$pin_root/usr/share/limine-entry-tool.d"
+  while (($#)); do
+    printf '%s\n' "$2" >>"$pin_root/$1"
+    shift 2
+  done
+  kernel_cmdline_root_pinned
+}
+root_pinned etc/default/limine 'KERNEL_CMDLINE[default]+=" root=UUID=abc rw"' || fail "a quoted root= pin is recognised"
+root_pinned etc/default/limine 'KERNEL_CMDLINE[default]=root=UUID=abc' || fail "an unquoted root= pin is recognised"
+root_pinned etc/default/limine 'KERNEL_CMDLINE[default]+=" dm-mod.create="foo" root=UUID=abc rw"' ||
+  fail "a pin with a quoted parameter before root= is recognised"
+root_pinned usr/share/limine-entry-tool.d/root.conf 'KERNEL_CMDLINE[default]+="root=UUID=abc"' \
+  etc/default/limine 'KERNEL_CMDLINE[default]+=" rw"' || fail "an appending layer keeps an earlier pin"
+root_pinned etc/default/limine 'KERNEL_CMDLINE[default] += " root=UUID=abc rw"' || fail "a pin spaced around += is recognised"
+! root_pinned etc/default/limine "KERNEL_CMDLINE[default]+='root=UUID=abc rw'" ||
+  fail "single quotes are kept, as limine-entry-tool keeps them"
+! root_pinned etc/default/limine 'OLD_KERNEL_CMDLINE[default]+=" root=UUID=abc rw"' || fail "a renamed key is not a pin"
+! root_pinned etc/default/limine '# KERNEL_CMDLINE[default]+=" root=UUID=abc rw"' || fail "a commented-out pin is not a pin"
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[fallback]+=" root=UUID=abc rw"' || fail "a fallback-only pin does not cover default"
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[default]+=" rootflags=subvol=@ rw"' || fail "rootflags= is not root="
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[default]+=" systemd.setenv="root=UUID=abc" rw"' ||
+  fail "root= inside another parameter's value is not a pin"
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[default]+=" systemd.setenv="NOTE=x root=UUID=abc" rw"' ||
+  fail "root= after a space inside a quoted value is not a pin"
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[default]=systemd.setenv="NOTE=x root=UUID=abc rw' ||
+  fail "root= after an unmatched quote is not a pin"
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[default]=ro"x"ot=UUID=abc rw' ||
+  fail "a quoted segment inside a parameter name does not make it root="
+! root_pinned etc/limine-entry-tool.conf 'KERNEL_CMDLINE[default]=root=UUID=abc rw' \
+  etc/default/limine 'KERNEL_CMDLINE[default]=quiet' || fail "a pin replaced by a later layer is not a pin"
+! root_pinned etc/default/limine 'KERNEL_CMDLINE[default]+=" root=UUID=abc rw"' \
+  etc/default/limine 'KERNEL_CMDLINE[default]="quiet"' || fail "a pin replaced later in the same layer is not a pin"
+unset -f as_root
+pass "Omarchy 4 upgrade recognises exactly the root= pins that hold"
+
+# Installing the limine packages can deploy limine on a machine that had no
+# limine.conf when the pin ran, so verification pins whatever the first call skipped.
+(
+  eval "$(sed -n '/^kernel_cmdline_root_checked=/p;/^preserve_kernel_cmdline_root() {$/,/^}$/p;/^verify_kernel_cmdline_root() {$/,/^}$/p' "$upgrade_to_quattro")"
+  limine_conf=0 pin_checks=0
+  as_root() {
+    if [[ $1 == "test" ]]; then
+      ((limine_conf))
+    fi
+  }
+  kernel_cmdline_root_pinned() { ((++pin_checks)); }
+  limine-mkinitcpio() { :; }
+  preserve_kernel_cmdline_root
+  ((pin_checks == 0)) || fail "the pin waits for a limine.conf"
+  limine_conf=1
+  verify_kernel_cmdline_root
+  ((pin_checks == 1)) || fail "kernel cmdline verification pins a machine the transaction put on limine"
+  verify_kernel_cmdline_root
+  ((pin_checks == 1)) || fail "kernel cmdline verification pins a machine only once"
+)
+pass "Omarchy 4 upgrade pins root= on machines the transaction moves to limine"
 
 # The crypt layer hides in the parents on LVM-on-LUKS, and a partial cmdline
 # for an encrypted root must not be written at all.