Clean up the terminal and reconnect when SSH connections drop (#6661)

* Clean up the terminal and reconnect when SSH connections drop

A remote tmux, herdr, or editor arms terminal modes over the SSH pipe
(mouse tracking, focus reporting, the alternate screen) that only it can
disarm. When the connection dies instead of exiting cleanly, those modes
stay armed on the local terminal, and every mouse move floods the prompt
with escape-sequence junk.

Wrap ssh in a shell function that disarms those modes after every exit,
and automatically reconnects when an established interactive session
drops. Remote commands, configured RemoteCommands, and redirected stdin
never reconnect, so their side effects cannot replay, and the retry loop
runs in a subshell so Ctrl-C cancels both the in-flight attempt and the
loop.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Detect dead SSH connections within a minute

Without keepalives, ssh does not notice a dead peer until TCP gives up,
which can take hours of sitting on a hung terminal with remote-armed
terminal modes stuck on. Ship a client keepalive default so drops are
detected in about 45 seconds, letting the shell's ssh wrapper clean up
and reconnect. ~/.ssh/config is read first and wins, so per-host
overrides still apply.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Fail closed when ssh -G cannot resolve the effective config

An unresolvable configuration could hide a RemoteCommand, so treat it
as non-interactive rather than reconnectable. Also strengthen the
tests from Copilot review: assert the complete disarm sequence, and
verify on a real interactive pty that Ctrl-C during a retry attempt
kills the reconnect loop itself, not just the in-flight attempt.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Tolerate the explicit RemoteCommand none when probing ssh -G

The literal "none" is how ssh_config cancels a configured
RemoteCommand, and some OpenSSH versions emit it even when unset, which
would have silently disabled reconnecting entirely. Treat it as no
remote command while still failing closed on real ones and unresolvable
configs, and make the fake ssh -G emit the "none" form so the behavior
tests cover it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

default/bash/fns/ssh-reconnect

@@ -0,0 +1,88 @@
+# Wrap ssh to clean up the terminal and reconnect when a connection drops.
+#
+# A remote tmux, herdr, or editor arms terminal modes over the SSH pipe (mouse
+# tracking, focus reporting, the alternate screen) that only it can disarm. If
+# the connection dies instead of exiting cleanly, those modes stay armed on the
+# local terminal, and every mouse move floods the prompt with escape junk.
+ssh() {
+  local rc started
+
+  started=$SECONDS
+  command ssh "$@"
+  rc=$?
+
+  [[ -t 1 ]] || return $rc
+  _ssh_disarm
+
+  # Reconnect only when an interactive session drops: ssh exits 255 for
+  # transport failures, but a fast 255 with no established session is a
+  # connect/auth failure, a remote command's own 255 passes through
+  # indistinguishably and must not replay its side effects, and redirected
+  # stdin would feed the remaining piped input to a fresh remote shell.
+  if (( rc != 255 )) || [[ ! -t 0 ]] || ! _ssh_interactive "$@" ||
+    (( SECONDS - started < 30 )); then
+    return $rc
+  fi
+
+  # Retry in a subshell: Ctrl-C reaches the whole foreground process group,
+  # so it cancels both the in-flight attempt and the loop itself. Keep
+  # retrying fast failures, since a rebooting server refuses connections too.
+  (
+    while true; do
+      echo "Connection lost. Reconnecting (Ctrl-C to stop)..."
+      sleep 2
+      command ssh "$@"
+      rc=$?
+      _ssh_disarm
+      (( rc != 255 )) && exit $rc
+    done
+  )
+}
+
+# Disarm mouse tracking (1000/1002/1003, 1006 encoding), focus reporting
+# (1004), and the alternate screen (1049), and show the cursor again.
+_ssh_disarm() {
+  printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1004l\e[?1049l\e[?25h'
+}
+
+# True for an interactive session: a destination and no remote command. The
+# letters are the ssh(1) options that consume a value, so their arguments are
+# not mistaken for the destination.
+_ssh_interactive() {
+  local value_opts="BbcDEeFIiJLlmOoPpQRSWw"
+  local argv=("$@") arg letters i dest="" opts_done=""
+
+  while (($#)); do
+    arg="$1"
+    shift
+
+    if [[ -z $opts_done && $arg == "--" ]]; then
+      opts_done=1
+    elif [[ -z $opts_done && $arg == -?* ]]; then
+      letters="${arg#-}"
+      for ((i = 0; i < ${#letters}; i++)); do
+        if [[ $value_opts == *"${letters:i:1}"* ]]; then
+          # The value is glued to the letter (-p2222) unless the letter ends
+          # the argument, in which case it consumes the next one (-p 2222).
+          (( i == ${#letters} - 1 )) && shift
+          break
+        fi
+      done
+    elif [[ -z $dest ]]; then
+      dest="$arg"
+    else
+      return 1
+    fi
+  done
+
+  [[ -n $dest ]] || return 1
+
+  # A RemoteCommand from ssh_config or -o replays on reconnect just like a
+  # positional command; ssh -G resolves the effective configuration for this
+  # exact invocation without connecting. Fail closed when it cannot resolve,
+  # since an undetected RemoteCommand must not replay. The explicit "none"
+  # cancels a configured command, and some versions emit it when unset.
+  local resolved
+  resolved=$(command ssh -G "${argv[@]}" 2>/dev/null) || return 1
+  ! grep -i '^remotecommand ' <<<"$resolved" | grep -qvi '^remotecommand none$'
+}

install/config/all.sh

@@ -3,6 +3,7 @@ run_logged "$OMARCHY_INSTALL/config/increase-lockout-limit.sh"
 run_logged "$OMARCHY_INSTALL/config/lockscreen-pam.sh"
 run_logged "$OMARCHY_INSTALL/config/fix-powerprofilesctl-shebang.sh"
 run_logged "$OMARCHY_INSTALL/config/ssh-command-path.sh"
+run_logged "$OMARCHY_INSTALL/config/ssh-keepalive.sh"
 run_logged "$OMARCHY_INSTALL/config/docker.sh"
 run_logged "$OMARCHY_INSTALL/config/snapper.sh"
 run_logged "$OMARCHY_INSTALL/config/locate.sh"

install/config/ssh-keepalive.sh

@@ -0,0 +1,17 @@
+# Without keepalives, ssh does not notice a dead connection until TCP gives up,
+# which can take hours of sitting on a hung terminal with any remote-armed
+# terminal modes stuck on. Detect the drop within a minute so the shell's ssh
+# wrapper can clean up and reconnect. ~/.ssh/config is read first and wins, so
+# users can still override these defaults per host.
+if [[ ! -f /etc/ssh/ssh_config.d/20-omarchy-keepalive.conf ]]; then
+  install -d -m 755 /etc/ssh/ssh_config.d
+  cat >/etc/ssh/ssh_config.d/20-omarchy-keepalive.conf <<'EOF'
+# Omarchy: notice dropped connections quickly instead of hanging until TCP
+# times out. Settings in ~/.ssh/config take precedence over these defaults.
+Host *
+  ServerAliveInterval 15
+  ServerAliveCountMax 3
+  ConnectTimeout 10
+EOF
+  chmod 644 /etc/ssh/ssh_config.d/20-omarchy-keepalive.conf
+fi

migrations/1786278735.sh

@@ -0,0 +1,18 @@
+echo "Detect dropped SSH connections quickly instead of leaving terminals hung"
+
+conf="/etc/ssh/ssh_config.d/20-omarchy-keepalive.conf"
+
+if [[ ! -f $conf ]]; then
+  sudo install -d -m 755 /etc/ssh/ssh_config.d
+  # An explicit mode so a restrictive user umask cannot leave the root-owned
+  # drop-in unreadable to the unprivileged ssh client.
+  sudo tee "$conf" >/dev/null <<'EOF'
+# Omarchy: notice dropped connections quickly instead of hanging until TCP
+# times out. Settings in ~/.ssh/config take precedence over these defaults.
+Host *
+  ServerAliveInterval 15
+  ServerAliveCountMax 3
+  ConnectTimeout 10
+EOF
+  sudo chmod 644 "$conf"
+fi

test/shell.d/ssh-reconnect-test.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"
+
+require_command script
+
+fns="$ROOT/default/bash/fns/ssh-reconnect"
+
+# The behavior tests patch the 30-second drop threshold down to 2 seconds so
+# they run quickly; make sure the marker they patch still exists.
+grep -q 'SECONDS - started < 30' "$fns" ||
+  fail "drop threshold marker exists for test patching"
+pass "drop threshold marker exists for test patching"
+
+# --- _ssh_interactive parsing ---
+
+run_interactive() {
+  bash -c "source '$fns'; _ssh_interactive \"\$@\"" _ "$@"
+}
+
+run_interactive -F /dev/null host || fail "plain destination is interactive"
+pass "plain destination is interactive"
+
+run_interactive -F /dev/null -p 2222 user@host || fail "separated option value is skipped"
+pass "separated option value is skipped"
+
+run_interactive -F /dev/null -4p2222 host || fail "glued option value is skipped"
+pass "glued option value is skipped"
+
+run_interactive -F /dev/null -L 8080:localhost:80 -o "ServerAliveInterval=5" host ||
+  fail "forwarding and -o options are skipped"
+pass "forwarding and -o options are skipped"
+
+if run_interactive -F /dev/null -o "RemoteCommand=uptime" host; then
+  fail "configured RemoteCommand is not interactive"
+fi
+pass "configured RemoteCommand is not interactive"
+
+run_interactive -F /dev/null -o "RemoteCommand=none" host ||
+  fail "explicit RemoteCommand none stays interactive"
+pass "explicit RemoteCommand none stays interactive"
+
+if run_interactive host uptime; then
+  fail "remote command is not interactive"
+fi
+pass "remote command is not interactive"
+
+if run_interactive -- host uptime; then
+  fail "remote command after -- is not interactive"
+fi
+pass "remote command after -- is not interactive"
+
+if run_interactive -p 2222; then
+  fail "missing destination is not interactive"
+fi
+pass "missing destination is not interactive"
+
+# --- reconnect behavior, run on a pty with a fake ssh ---
+
+fake_dir=$(mktemp -d)
+trap 'rm -rf "$fake_dir"' EXIT
+
+cat >"$fake_dir/ssh" <<EOF
+#!/bin/bash
+[[ \$1 == "-G" ]] && { echo "remotecommand none"; exit 0; }
+n=\$(( \$(cat "$fake_dir/count" 2>/dev/null || echo 0) + 1 ))
+echo "\$n" >"$fake_dir/count"
+read -r duration code <<<"\$(sed -n "\${n}p" "$fake_dir/plan")"
+sleep "\$duration"
+exit "\$code"
+EOF
+chmod +x "$fake_dir/ssh"
+
+cat >"$fake_dir/driver" <<EOF
+PATH="$fake_dir:\$PATH"
+source <(sed 's/< 30/< 2/' "$fns")
+sleep() { :; }
+ssh "\$@"
+echo "rc=\$?"
+EOF
+
+# Each plan line is "<duration> <exit code>" for one fake ssh attempt.
+run_case() {
+  local plan="$1"
+  shift
+  printf '%s\n' "$plan" >"$fake_dir/plan"
+  rm -f "$fake_dir/count"
+  script -qec "bash '$fake_dir/driver' $*" /dev/null | tr -d '\r'
+}
+
+attempts() {
+  cat "$fake_dir/count"
+}
+
+disarm=$(printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1004l\e[?1049l\e[?25h')
+
+out=$(run_case "0 255" host)
+[[ $out == *"$disarm"* ]] || fail "stray terminal modes are reset after ssh exits" "$out"
+pass "stray terminal modes are reset after ssh exits"
+
+[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) ||
+  fail "fast connection failure does not reconnect" "$out"
+pass "fast connection failure does not reconnect"
+
+out=$(run_case $'2 255\n0 0' host)
+[[ $out == *"Connection lost"* ]] && [[ $out == *"rc=0"* ]] && (( $(attempts) == 2 )) ||
+  fail "dropped session reconnects" "$out"
+pass "dropped session reconnects"
+
+out=$(run_case $'2 255\n0 255\n0 0' host)
+[[ $out == *"rc=0"* ]] && (( $(attempts) == 3 )) ||
+  fail "reconnecting keeps retrying while the server is still down" "$out"
+pass "reconnecting keeps retrying while the server is still down"
+
+out=$(run_case $'2 255\n0 0' host uptime)
+[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) ||
+  fail "remote command exiting 255 is not replayed" "$out"
+pass "remote command exiting 255 is not replayed"
+
+printf '%s\n' $'2 255\n0 0' >"$fake_dir/plan"
+rm -f "$fake_dir/count"
+out=$(script -qec "bash '$fake_dir/driver' host </dev/null" /dev/null | tr -d '\r')
+[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) ||
+  fail "redirected stdin does not reconnect" "$out"
+pass "redirected stdin does not reconnect"
+
+# Ctrl-C must stop the loop, not just the in-flight attempt. Only a real
+# interactive shell reproduces the job-control process groups this depends on,
+# so type the command, the ^C, and the status check into one over a pty. With
+# the loop killed, attempts stay low; a surviving loop would drain the whole
+# plan before reporting a different status.
+cat >"$fake_dir/rcfile" <<EOF
+PATH="$fake_dir:\$PATH"
+source <(sed 's/< 30/< 2/' "$fns")
+sleep() { :; }
+PS1='$ '
+EOF
+
+plan="2 255"
+for _ in {1..12}; do plan+=$'\n1 255'; done
+printf '%s\n' "$plan" >"$fake_dir/plan"
+rm -f "$fake_dir/count"
+out=$({
+  printf 'ssh host\n'
+  command sleep 4
+  printf '\003'
+  command sleep 1
+  printf 'echo DONE rc=$?\n'
+  command sleep 1
+  printf 'exit\n'
+} | script -qec "bash --rcfile '$fake_dir/rcfile' -i" /dev/null | tr -d '\r')
+[[ $out == *"Connection lost"* ]] && [[ $out == *"DONE rc=130"* ]] && (( $(attempts) < 10 )) ||
+  fail "Ctrl-C during a retry attempt stops the reconnect loop" "$out"
+pass "Ctrl-C during a retry attempt stops the reconnect loop"