diff --git a/default/bash/fns/ssh-reconnect b/default/bash/fns/ssh-reconnect new file mode 100644 index 00000000..5a342b63 --- /dev/null +++ b/default/bash/fns/ssh-reconnect @@ -0,0 +1,88 @@ +# Wrap ssh to clean up the terminal and reconnect when a connection drops. +# +# A remote tmux, herdr, or editor arms terminal modes over the SSH pipe (mouse +# tracking, focus reporting, the alternate screen) that only it can disarm. If +# the connection dies instead of exiting cleanly, those modes stay armed on the +# local terminal, and every mouse move floods the prompt with escape junk. +ssh() { + local rc started + + started=$SECONDS + command ssh "$@" + rc=$? + + [[ -t 1 ]] || return $rc + _ssh_disarm + + # Reconnect only when an interactive session drops: ssh exits 255 for + # transport failures, but a fast 255 with no established session is a + # connect/auth failure, a remote command's own 255 passes through + # indistinguishably and must not replay its side effects, and redirected + # stdin would feed the remaining piped input to a fresh remote shell. + if (( rc != 255 )) || [[ ! -t 0 ]] || ! _ssh_interactive "$@" || + (( SECONDS - started < 30 )); then + return $rc + fi + + # Retry in a subshell: Ctrl-C reaches the whole foreground process group, + # so it cancels both the in-flight attempt and the loop itself. Keep + # retrying fast failures, since a rebooting server refuses connections too. + ( + while true; do + echo "Connection lost. Reconnecting (Ctrl-C to stop)..." + sleep 2 + command ssh "$@" + rc=$? + _ssh_disarm + (( rc != 255 )) && exit $rc + done + ) +} + +# Disarm mouse tracking (1000/1002/1003, 1006 encoding), focus reporting +# (1004), and the alternate screen (1049), and show the cursor again. +_ssh_disarm() { + printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1004l\e[?1049l\e[?25h' +} + +# True for an interactive session: a destination and no remote command. The +# letters are the ssh(1) options that consume a value, so their arguments are +# not mistaken for the destination. +_ssh_interactive() { + local value_opts="BbcDEeFIiJLlmOoPpQRSWw" + local argv=("$@") arg letters i dest="" opts_done="" + + while (($#)); do + arg="$1" + shift + + if [[ -z $opts_done && $arg == "--" ]]; then + opts_done=1 + elif [[ -z $opts_done && $arg == -?* ]]; then + letters="${arg#-}" + for ((i = 0; i < ${#letters}; i++)); do + if [[ $value_opts == *"${letters:i:1}"* ]]; then + # The value is glued to the letter (-p2222) unless the letter ends + # the argument, in which case it consumes the next one (-p 2222). + (( i == ${#letters} - 1 )) && shift + break + fi + done + elif [[ -z $dest ]]; then + dest="$arg" + else + return 1 + fi + done + + [[ -n $dest ]] || return 1 + + # A RemoteCommand from ssh_config or -o replays on reconnect just like a + # positional command; ssh -G resolves the effective configuration for this + # exact invocation without connecting. Fail closed when it cannot resolve, + # since an undetected RemoteCommand must not replay. The explicit "none" + # cancels a configured command, and some versions emit it when unset. + local resolved + resolved=$(command ssh -G "${argv[@]}" 2>/dev/null) || return 1 + ! grep -i '^remotecommand ' <<<"$resolved" | grep -qvi '^remotecommand none$' +} diff --git a/install/config/all.sh b/install/config/all.sh index ba2247b9..91256dc7 100644 --- a/install/config/all.sh +++ b/install/config/all.sh @@ -3,6 +3,7 @@ run_logged "$OMARCHY_INSTALL/config/increase-lockout-limit.sh" run_logged "$OMARCHY_INSTALL/config/lockscreen-pam.sh" run_logged "$OMARCHY_INSTALL/config/fix-powerprofilesctl-shebang.sh" run_logged "$OMARCHY_INSTALL/config/ssh-command-path.sh" +run_logged "$OMARCHY_INSTALL/config/ssh-keepalive.sh" run_logged "$OMARCHY_INSTALL/config/docker.sh" run_logged "$OMARCHY_INSTALL/config/snapper.sh" run_logged "$OMARCHY_INSTALL/config/locate.sh" diff --git a/install/config/ssh-keepalive.sh b/install/config/ssh-keepalive.sh new file mode 100644 index 00000000..b91d6cbb --- /dev/null +++ b/install/config/ssh-keepalive.sh @@ -0,0 +1,17 @@ +# Without keepalives, ssh does not notice a dead connection until TCP gives up, +# which can take hours of sitting on a hung terminal with any remote-armed +# terminal modes stuck on. Detect the drop within a minute so the shell's ssh +# wrapper can clean up and reconnect. ~/.ssh/config is read first and wins, so +# users can still override these defaults per host. +if [[ ! -f /etc/ssh/ssh_config.d/20-omarchy-keepalive.conf ]]; then + install -d -m 755 /etc/ssh/ssh_config.d + cat >/etc/ssh/ssh_config.d/20-omarchy-keepalive.conf <<'EOF' +# Omarchy: notice dropped connections quickly instead of hanging until TCP +# times out. Settings in ~/.ssh/config take precedence over these defaults. +Host * + ServerAliveInterval 15 + ServerAliveCountMax 3 + ConnectTimeout 10 +EOF + chmod 644 /etc/ssh/ssh_config.d/20-omarchy-keepalive.conf +fi diff --git a/migrations/1786278735.sh b/migrations/1786278735.sh new file mode 100644 index 00000000..291723fe --- /dev/null +++ b/migrations/1786278735.sh @@ -0,0 +1,18 @@ +echo "Detect dropped SSH connections quickly instead of leaving terminals hung" + +conf="/etc/ssh/ssh_config.d/20-omarchy-keepalive.conf" + +if [[ ! -f $conf ]]; then + sudo install -d -m 755 /etc/ssh/ssh_config.d + # An explicit mode so a restrictive user umask cannot leave the root-owned + # drop-in unreadable to the unprivileged ssh client. + sudo tee "$conf" >/dev/null <<'EOF' +# Omarchy: notice dropped connections quickly instead of hanging until TCP +# times out. Settings in ~/.ssh/config take precedence over these defaults. +Host * + ServerAliveInterval 15 + ServerAliveCountMax 3 + ConnectTimeout 10 +EOF + sudo chmod 644 "$conf" +fi diff --git a/test/shell.d/ssh-reconnect-test.sh b/test/shell.d/ssh-reconnect-test.sh new file mode 100755 index 00000000..06b15758 --- /dev/null +++ b/test/shell.d/ssh-reconnect-test.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +set -euo pipefail + +source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh" + +require_command script + +fns="$ROOT/default/bash/fns/ssh-reconnect" + +# The behavior tests patch the 30-second drop threshold down to 2 seconds so +# they run quickly; make sure the marker they patch still exists. +grep -q 'SECONDS - started < 30' "$fns" || + fail "drop threshold marker exists for test patching" +pass "drop threshold marker exists for test patching" + +# --- _ssh_interactive parsing --- + +run_interactive() { + bash -c "source '$fns'; _ssh_interactive \"\$@\"" _ "$@" +} + +run_interactive -F /dev/null host || fail "plain destination is interactive" +pass "plain destination is interactive" + +run_interactive -F /dev/null -p 2222 user@host || fail "separated option value is skipped" +pass "separated option value is skipped" + +run_interactive -F /dev/null -4p2222 host || fail "glued option value is skipped" +pass "glued option value is skipped" + +run_interactive -F /dev/null -L 8080:localhost:80 -o "ServerAliveInterval=5" host || + fail "forwarding and -o options are skipped" +pass "forwarding and -o options are skipped" + +if run_interactive -F /dev/null -o "RemoteCommand=uptime" host; then + fail "configured RemoteCommand is not interactive" +fi +pass "configured RemoteCommand is not interactive" + +run_interactive -F /dev/null -o "RemoteCommand=none" host || + fail "explicit RemoteCommand none stays interactive" +pass "explicit RemoteCommand none stays interactive" + +if run_interactive host uptime; then + fail "remote command is not interactive" +fi +pass "remote command is not interactive" + +if run_interactive -- host uptime; then + fail "remote command after -- is not interactive" +fi +pass "remote command after -- is not interactive" + +if run_interactive -p 2222; then + fail "missing destination is not interactive" +fi +pass "missing destination is not interactive" + +# --- reconnect behavior, run on a pty with a fake ssh --- + +fake_dir=$(mktemp -d) +trap 'rm -rf "$fake_dir"' EXIT + +cat >"$fake_dir/ssh" </dev/null || echo 0) + 1 )) +echo "\$n" >"$fake_dir/count" +read -r duration code <<<"\$(sed -n "\${n}p" "$fake_dir/plan")" +sleep "\$duration" +exit "\$code" +EOF +chmod +x "$fake_dir/ssh" + +cat >"$fake_dir/driver" < " for one fake ssh attempt. +run_case() { + local plan="$1" + shift + printf '%s\n' "$plan" >"$fake_dir/plan" + rm -f "$fake_dir/count" + script -qec "bash '$fake_dir/driver' $*" /dev/null | tr -d '\r' +} + +attempts() { + cat "$fake_dir/count" +} + +disarm=$(printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1004l\e[?1049l\e[?25h') + +out=$(run_case "0 255" host) +[[ $out == *"$disarm"* ]] || fail "stray terminal modes are reset after ssh exits" "$out" +pass "stray terminal modes are reset after ssh exits" + +[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) || + fail "fast connection failure does not reconnect" "$out" +pass "fast connection failure does not reconnect" + +out=$(run_case $'2 255\n0 0' host) +[[ $out == *"Connection lost"* ]] && [[ $out == *"rc=0"* ]] && (( $(attempts) == 2 )) || + fail "dropped session reconnects" "$out" +pass "dropped session reconnects" + +out=$(run_case $'2 255\n0 255\n0 0' host) +[[ $out == *"rc=0"* ]] && (( $(attempts) == 3 )) || + fail "reconnecting keeps retrying while the server is still down" "$out" +pass "reconnecting keeps retrying while the server is still down" + +out=$(run_case $'2 255\n0 0' host uptime) +[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) || + fail "remote command exiting 255 is not replayed" "$out" +pass "remote command exiting 255 is not replayed" + +printf '%s\n' $'2 255\n0 0' >"$fake_dir/plan" +rm -f "$fake_dir/count" +out=$(script -qec "bash '$fake_dir/driver' host "$fake_dir/rcfile" <"$fake_dir/plan" +rm -f "$fake_dir/count" +out=$({ + printf 'ssh host\n' + command sleep 4 + printf '\003' + command sleep 1 + printf 'echo DONE rc=$?\n' + command sleep 1 + printf 'exit\n' +} | script -qec "bash --rcfile '$fake_dir/rcfile' -i" /dev/null | tr -d '\r') +[[ $out == *"Connection lost"* ]] && [[ $out == *"DONE rc=130"* ]] && (( $(attempts) < 10 )) || + fail "Ctrl-C during a retry attempt stops the reconnect loop" "$out" +pass "Ctrl-C during a retry attempt stops the reconnect loop"