From 6ddc39520d5387e3b88cd74087ae44e5fbad83d7 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Sun, 9 Aug 2026 19:15:21 +0200 Subject: [PATCH] Clean up the terminal and reconnect when SSH connections drop (#6661) * Clean up the terminal and reconnect when SSH connections drop A remote tmux, herdr, or editor arms terminal modes over the SSH pipe (mouse tracking, focus reporting, the alternate screen) that only it can disarm. When the connection dies instead of exiting cleanly, those modes stay armed on the local terminal, and every mouse move floods the prompt with escape-sequence junk. Wrap ssh in a shell function that disarms those modes after every exit, and automatically reconnects when an established interactive session drops. Remote commands, configured RemoteCommands, and redirected stdin never reconnect, so their side effects cannot replay, and the retry loop runs in a subshell so Ctrl-C cancels both the in-flight attempt and the loop. Co-Authored-By: Claude Fable 5 * Detect dead SSH connections within a minute Without keepalives, ssh does not notice a dead peer until TCP gives up, which can take hours of sitting on a hung terminal with remote-armed terminal modes stuck on. Ship a client keepalive default so drops are detected in about 45 seconds, letting the shell's ssh wrapper clean up and reconnect. ~/.ssh/config is read first and wins, so per-host overrides still apply. Co-Authored-By: Claude Fable 5 * Fail closed when ssh -G cannot resolve the effective config An unresolvable configuration could hide a RemoteCommand, so treat it as non-interactive rather than reconnectable. Also strengthen the tests from Copilot review: assert the complete disarm sequence, and verify on a real interactive pty that Ctrl-C during a retry attempt kills the reconnect loop itself, not just the in-flight attempt. Co-Authored-By: Claude Fable 5 * Tolerate the explicit RemoteCommand none when probing ssh -G The literal "none" is how ssh_config cancels a configured RemoteCommand, and some OpenSSH versions emit it even when unset, which would have silently disabled reconnecting entirely. Treat it as no remote command while still failing closed on real ones and unresolvable configs, and make the fake ssh -G emit the "none" form so the behavior tests cover it. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- default/bash/fns/ssh-reconnect | 88 ++++++++++++++++ install/config/all.sh | 1 + install/config/ssh-keepalive.sh | 17 ++++ migrations/1786278735.sh | 18 ++++ test/shell.d/ssh-reconnect-test.sh | 156 +++++++++++++++++++++++++++++ 5 files changed, 280 insertions(+) create mode 100644 default/bash/fns/ssh-reconnect create mode 100644 install/config/ssh-keepalive.sh create mode 100644 migrations/1786278735.sh create mode 100755 test/shell.d/ssh-reconnect-test.sh diff --git a/default/bash/fns/ssh-reconnect b/default/bash/fns/ssh-reconnect new file mode 100644 index 00000000..5a342b63 --- /dev/null +++ b/default/bash/fns/ssh-reconnect @@ -0,0 +1,88 @@ +# Wrap ssh to clean up the terminal and reconnect when a connection drops. +# +# A remote tmux, herdr, or editor arms terminal modes over the SSH pipe (mouse +# tracking, focus reporting, the alternate screen) that only it can disarm. If +# the connection dies instead of exiting cleanly, those modes stay armed on the +# local terminal, and every mouse move floods the prompt with escape junk. +ssh() { + local rc started + + started=$SECONDS + command ssh "$@" + rc=$? + + [[ -t 1 ]] || return $rc + _ssh_disarm + + # Reconnect only when an interactive session drops: ssh exits 255 for + # transport failures, but a fast 255 with no established session is a + # connect/auth failure, a remote command's own 255 passes through + # indistinguishably and must not replay its side effects, and redirected + # stdin would feed the remaining piped input to a fresh remote shell. + if (( rc != 255 )) || [[ ! -t 0 ]] || ! _ssh_interactive "$@" || + (( SECONDS - started < 30 )); then + return $rc + fi + + # Retry in a subshell: Ctrl-C reaches the whole foreground process group, + # so it cancels both the in-flight attempt and the loop itself. Keep + # retrying fast failures, since a rebooting server refuses connections too. + ( + while true; do + echo "Connection lost. Reconnecting (Ctrl-C to stop)..." + sleep 2 + command ssh "$@" + rc=$? + _ssh_disarm + (( rc != 255 )) && exit $rc + done + ) +} + +# Disarm mouse tracking (1000/1002/1003, 1006 encoding), focus reporting +# (1004), and the alternate screen (1049), and show the cursor again. +_ssh_disarm() { + printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1004l\e[?1049l\e[?25h' +} + +# True for an interactive session: a destination and no remote command. The +# letters are the ssh(1) options that consume a value, so their arguments are +# not mistaken for the destination. +_ssh_interactive() { + local value_opts="BbcDEeFIiJLlmOoPpQRSWw" + local argv=("$@") arg letters i dest="" opts_done="" + + while (($#)); do + arg="$1" + shift + + if [[ -z $opts_done && $arg == "--" ]]; then + opts_done=1 + elif [[ -z $opts_done && $arg == -?* ]]; then + letters="${arg#-}" + for ((i = 0; i < ${#letters}; i++)); do + if [[ $value_opts == *"${letters:i:1}"* ]]; then + # The value is glued to the letter (-p2222) unless the letter ends + # the argument, in which case it consumes the next one (-p 2222). + (( i == ${#letters} - 1 )) && shift + break + fi + done + elif [[ -z $dest ]]; then + dest="$arg" + else + return 1 + fi + done + + [[ -n $dest ]] || return 1 + + # A RemoteCommand from ssh_config or -o replays on reconnect just like a + # positional command; ssh -G resolves the effective configuration for this + # exact invocation without connecting. Fail closed when it cannot resolve, + # since an undetected RemoteCommand must not replay. The explicit "none" + # cancels a configured command, and some versions emit it when unset. + local resolved + resolved=$(command ssh -G "${argv[@]}" 2>/dev/null) || return 1 + ! grep -i '^remotecommand ' <<<"$resolved" | grep -qvi '^remotecommand none$' +} diff --git a/install/config/all.sh b/install/config/all.sh index ba2247b9..91256dc7 100644 --- a/install/config/all.sh +++ b/install/config/all.sh @@ -3,6 +3,7 @@ run_logged "$OMARCHY_INSTALL/config/increase-lockout-limit.sh" run_logged "$OMARCHY_INSTALL/config/lockscreen-pam.sh" run_logged "$OMARCHY_INSTALL/config/fix-powerprofilesctl-shebang.sh" run_logged "$OMARCHY_INSTALL/config/ssh-command-path.sh" +run_logged "$OMARCHY_INSTALL/config/ssh-keepalive.sh" run_logged "$OMARCHY_INSTALL/config/docker.sh" run_logged "$OMARCHY_INSTALL/config/snapper.sh" run_logged "$OMARCHY_INSTALL/config/locate.sh" diff --git a/install/config/ssh-keepalive.sh b/install/config/ssh-keepalive.sh new file mode 100644 index 00000000..b91d6cbb --- /dev/null +++ b/install/config/ssh-keepalive.sh @@ -0,0 +1,17 @@ +# Without keepalives, ssh does not notice a dead connection until TCP gives up, +# which can take hours of sitting on a hung terminal with any remote-armed +# terminal modes stuck on. Detect the drop within a minute so the shell's ssh +# wrapper can clean up and reconnect. ~/.ssh/config is read first and wins, so +# users can still override these defaults per host. +if [[ ! -f /etc/ssh/ssh_config.d/20-omarchy-keepalive.conf ]]; then + install -d -m 755 /etc/ssh/ssh_config.d + cat >/etc/ssh/ssh_config.d/20-omarchy-keepalive.conf <<'EOF' +# Omarchy: notice dropped connections quickly instead of hanging until TCP +# times out. Settings in ~/.ssh/config take precedence over these defaults. +Host * + ServerAliveInterval 15 + ServerAliveCountMax 3 + ConnectTimeout 10 +EOF + chmod 644 /etc/ssh/ssh_config.d/20-omarchy-keepalive.conf +fi diff --git a/migrations/1786278735.sh b/migrations/1786278735.sh new file mode 100644 index 00000000..291723fe --- /dev/null +++ b/migrations/1786278735.sh @@ -0,0 +1,18 @@ +echo "Detect dropped SSH connections quickly instead of leaving terminals hung" + +conf="/etc/ssh/ssh_config.d/20-omarchy-keepalive.conf" + +if [[ ! -f $conf ]]; then + sudo install -d -m 755 /etc/ssh/ssh_config.d + # An explicit mode so a restrictive user umask cannot leave the root-owned + # drop-in unreadable to the unprivileged ssh client. + sudo tee "$conf" >/dev/null <<'EOF' +# Omarchy: notice dropped connections quickly instead of hanging until TCP +# times out. Settings in ~/.ssh/config take precedence over these defaults. +Host * + ServerAliveInterval 15 + ServerAliveCountMax 3 + ConnectTimeout 10 +EOF + sudo chmod 644 "$conf" +fi diff --git a/test/shell.d/ssh-reconnect-test.sh b/test/shell.d/ssh-reconnect-test.sh new file mode 100755 index 00000000..06b15758 --- /dev/null +++ b/test/shell.d/ssh-reconnect-test.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +set -euo pipefail + +source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh" + +require_command script + +fns="$ROOT/default/bash/fns/ssh-reconnect" + +# The behavior tests patch the 30-second drop threshold down to 2 seconds so +# they run quickly; make sure the marker they patch still exists. +grep -q 'SECONDS - started < 30' "$fns" || + fail "drop threshold marker exists for test patching" +pass "drop threshold marker exists for test patching" + +# --- _ssh_interactive parsing --- + +run_interactive() { + bash -c "source '$fns'; _ssh_interactive \"\$@\"" _ "$@" +} + +run_interactive -F /dev/null host || fail "plain destination is interactive" +pass "plain destination is interactive" + +run_interactive -F /dev/null -p 2222 user@host || fail "separated option value is skipped" +pass "separated option value is skipped" + +run_interactive -F /dev/null -4p2222 host || fail "glued option value is skipped" +pass "glued option value is skipped" + +run_interactive -F /dev/null -L 8080:localhost:80 -o "ServerAliveInterval=5" host || + fail "forwarding and -o options are skipped" +pass "forwarding and -o options are skipped" + +if run_interactive -F /dev/null -o "RemoteCommand=uptime" host; then + fail "configured RemoteCommand is not interactive" +fi +pass "configured RemoteCommand is not interactive" + +run_interactive -F /dev/null -o "RemoteCommand=none" host || + fail "explicit RemoteCommand none stays interactive" +pass "explicit RemoteCommand none stays interactive" + +if run_interactive host uptime; then + fail "remote command is not interactive" +fi +pass "remote command is not interactive" + +if run_interactive -- host uptime; then + fail "remote command after -- is not interactive" +fi +pass "remote command after -- is not interactive" + +if run_interactive -p 2222; then + fail "missing destination is not interactive" +fi +pass "missing destination is not interactive" + +# --- reconnect behavior, run on a pty with a fake ssh --- + +fake_dir=$(mktemp -d) +trap 'rm -rf "$fake_dir"' EXIT + +cat >"$fake_dir/ssh" </dev/null || echo 0) + 1 )) +echo "\$n" >"$fake_dir/count" +read -r duration code <<<"\$(sed -n "\${n}p" "$fake_dir/plan")" +sleep "\$duration" +exit "\$code" +EOF +chmod +x "$fake_dir/ssh" + +cat >"$fake_dir/driver" < " for one fake ssh attempt. +run_case() { + local plan="$1" + shift + printf '%s\n' "$plan" >"$fake_dir/plan" + rm -f "$fake_dir/count" + script -qec "bash '$fake_dir/driver' $*" /dev/null | tr -d '\r' +} + +attempts() { + cat "$fake_dir/count" +} + +disarm=$(printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1004l\e[?1049l\e[?25h') + +out=$(run_case "0 255" host) +[[ $out == *"$disarm"* ]] || fail "stray terminal modes are reset after ssh exits" "$out" +pass "stray terminal modes are reset after ssh exits" + +[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) || + fail "fast connection failure does not reconnect" "$out" +pass "fast connection failure does not reconnect" + +out=$(run_case $'2 255\n0 0' host) +[[ $out == *"Connection lost"* ]] && [[ $out == *"rc=0"* ]] && (( $(attempts) == 2 )) || + fail "dropped session reconnects" "$out" +pass "dropped session reconnects" + +out=$(run_case $'2 255\n0 255\n0 0' host) +[[ $out == *"rc=0"* ]] && (( $(attempts) == 3 )) || + fail "reconnecting keeps retrying while the server is still down" "$out" +pass "reconnecting keeps retrying while the server is still down" + +out=$(run_case $'2 255\n0 0' host uptime) +[[ $out == *"rc=255"* ]] && (( $(attempts) == 1 )) || + fail "remote command exiting 255 is not replayed" "$out" +pass "remote command exiting 255 is not replayed" + +printf '%s\n' $'2 255\n0 0' >"$fake_dir/plan" +rm -f "$fake_dir/count" +out=$(script -qec "bash '$fake_dir/driver' host "$fake_dir/rcfile" <"$fake_dir/plan" +rm -f "$fake_dir/count" +out=$({ + printf 'ssh host\n' + command sleep 4 + printf '\003' + command sleep 1 + printf 'echo DONE rc=$?\n' + command sleep 1 + printf 'exit\n' +} | script -qec "bash --rcfile '$fake_dir/rcfile' -i" /dev/null | tr -d '\r') +[[ $out == *"Connection lost"* ]] && [[ $out == *"DONE rc=130"* ]] && (( $(attempts) < 10 )) || + fail "Ctrl-C during a retry attempt stops the reconnect loop" "$out" +pass "Ctrl-C during a retry attempt stops the reconnect loop"