Report skipped checks separately in shell tests

2026-09-21 11:49:32 +02:00
1 parent 961ec7f39f
commit 599a6a665b
@@ -43,7 +43,7 @@ if (( EUID != 0 )); then
 fi

 if (( ! root_runtime_available )); then
-  pass "no unprivileged user namespace; skipping the root lock-helper lookup matrix"
+  skip "no unprivileged user namespace; skipping the root lock-helper lookup matrix"
  exit 0
 fi

@@ -7,7 +7,7 @@ source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"
 require_compositor "bar icon geometry test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping bar icon geometry test"
+  skip "quickshell not installed; skipping bar icon geometry test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "bar widget contract test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping bar widget contract test"
+  skip "quickshell not installed; skipping bar widget contract test"
  exit 0
 fi

@@ -14,6 +14,10 @@ pass() {
  printf 'ok - %s\n' "$1"
 }

+skip() {
+  printf 'ok - %s # SKIP\n' "$1"
+}
+
 fail() {
  local description="$1"
  local detail="${2:-}"
@@ -72,7 +76,7 @@ require_compositor() {
    return 0
  fi

-  pass "no Wayland compositor; skipping $description"
+  skip "no Wayland compositor; skipping $description"
  exit 0
 }

@@ -123,7 +123,7 @@ if [[ -n $real_hiddev ]]; then
    fail "wrapper did not trust a valid cached hiddev node: $real_hiddev"
  pass "wrapper trusts a cached hiddev character device without re-detecting"
 else
-  pass "no /dev/hiddev* character device present; skipping the valid-cache case"
+  skip "no /dev/hiddev* character device present; skipping the valid-cache case"
 fi

 # --- With no XDG_RUNTIME_DIR, the predictable /tmp cache is not consulted ------
@@ -148,5 +148,5 @@ if mkfifo "$tmp_cache" 2>/dev/null; then
      "it blocked reading the FIFO decoy at $tmp_cache"
  pass "wrapper ignores the /tmp cache path when XDG_RUNTIME_DIR is unset"
 else
-  pass "$tmp_cache already present or not safely creatable; skipping the /tmp-fallback case"
+  skip "$tmp_cache already present or not safely creatable; skipping the /tmp-fallback case"
 fi
@@ -46,7 +46,7 @@ mode=$(stat -c '%a' "$write_dir/color.json")
 pass "theme colour writes a 0644 color.json"

 if (( EUID == 0 )); then
-  pass "running as root; skipping the mktemp-failure check"
+  skip "running as root; skipping the mktemp-failure check"
 else
  chmod u+w "$write_dir"
  export TMPDIR=$test_tmp/missing-tmp
@@ -82,7 +82,7 @@ SH
 chmod +x "$stub_bin/sudo"

 if ((EUID == 0)); then
-  pass "running as root; skipping the elevation checks, which would rewrite this machine's browser policy"
+  skip "running as root; skipping the elevation checks, which would rewrite this machine's browser policy"
 else
  elevation_for() {
    : >"$test_tmp/elevation"
@@ -20,7 +20,7 @@ JS
 require_compositor "Button hover geometry runtime test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping Button hover geometry runtime test"
+  skip "quickshell not installed; skipping Button hover geometry runtime test"
  exit 0
 fi

@@ -6,7 +6,7 @@ source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"

 # Exercise the privileged installer without writing to the host's /usr/share.
 if ! command -v bwrap >/dev/null || ! bwrap --ro-bind / / --unshare-user --uid 0 --gid 0 true 2>/dev/null; then
-  pass "user namespaces unavailable; skipping isolated Claude extension installation"
+  skip "user namespaces unavailable; skipping isolated Claude extension installation"
  exit 0
 fi

@@ -62,7 +62,7 @@ run_guard() {
  ' bash "$SHELL_TEST_DIR" 2>&1 || printf 'guard exited %s\n' "$?"
 }

-skipped="ok - no Wayland compositor; skipping sample runtime test"
+skipped="ok - no Wayland compositor; skipping sample runtime test # SKIP"

 output=$(run_guard -u WAYLAND_DISPLAY XDG_RUNTIME_DIR="$runtime_dir")
 [[ $output == "$skipped" ]] || fail "guard skips without a display" "$output"
@@ -76,7 +76,7 @@ pass "guard skips when the socket is unreachable"

 # Everything below needs a socket to stand in for a live or abandoned compositor.
 if (( ! socket_bound )); then
-  pass "cannot bind a Unix socket here; skipping the cases that need one"
+  skip "cannot bind a Unix socket here; skipping the cases that need one"
  exit 0
 fi

@@ -81,13 +81,13 @@ SH
  rm -rf "$poison_dir"
  pass "root omarchy-dns resolves system helpers from a trusted PATH, not the invocation PATH"
 else
-  pass "no unprivileged user namespace; skipping the root trusted-PATH probe"
+  skip "no unprivileged user namespace; skipping the root trusted-PATH probe"
 fi

 # require_root returns immediately for root, so the stubs below would not stand
 # between the script and the host's real NetworkManager and resolved config.
 if (( EUID == 0 )); then
-  pass "running as root; skipping the elevation checks, which would rewrite this machine's DNS"
+  skip "running as root; skipping the elevation checks, which would rewrite this machine's DNS"
  exit 0
 fi

@@ -8,7 +8,7 @@ if (( EUID != 0 )); then
  if unshare --user --map-root-user true 2>/dev/null; then
    exec unshare --user --map-root-user bash "$0"
  fi
-  pass "no unprivileged user namespace; skipping factory account cleanup"
+  skip "no unprivileged user namespace; skipping factory account cleanup"
  exit 0
 fi

@@ -4,7 +4,7 @@ source "$(dirname "$0")/base-test.sh"

 # The hook guards on the real /etc/pam.d path, which can't be mocked via PATH.
 if [[ -f /etc/pam.d/omarchy-lock-fingerprint ]]; then
-  pass "fingerprint invitation test skipped: host already has fingerprint auth configured"
+  skip "fingerprint invitation test skipped: host already has fingerprint auth configured"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "QML contract test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping QML contract test"
+  skip "quickshell not installed; skipping QML contract test"
  exit 0
 fi

@@ -150,5 +150,5 @@ pass "font controls create missing Kitty overrides without restoring the theme i
 if "$ROOT/bin/omarchy-cmd-present" kitty; then
  kitty +runpy "$(cat "$ROOT/test/shell.d/fixtures/kitty/check-config.py")"
 else
-  pass "Kitty not installed; skipping native config parser checks"
+  skip "Kitty not installed; skipping native config parser checks"
 fi
@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "lock fingerprint indicator test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping lock fingerprint indicator test"
+  skip "quickshell not installed; skipping lock fingerprint indicator test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "lock password overflow test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping lock password overflow test"
+  skip "quickshell not installed; skipping lock password overflow test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "manifest entrypoint load test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping manifest entrypoint load test"
+  skip "quickshell not installed; skipping manifest entrypoint load test"
  exit 0
 fi

@@ -3,7 +3,7 @@ set -euo pipefail
 source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell unavailable; skipping lock feed QML lifecycle"
+  skip "quickshell unavailable; skipping lock feed QML lifecycle"
  exit 0
 fi
 work=$(mktemp -d)
@@ -158,7 +158,7 @@ if script -qec true /dev/null >/dev/null 2>&1; then
    fail "plugin add reached git clone for an option-shaped gum-prompt URL"
  pass "plugin add guard rejects an option-shaped URL from the interactive prompt"
 else
-  pass "script -qec unavailable; skipping the interactive gum-prompt guard case"
+  skip "script -qec unavailable; skipping the interactive gum-prompt guard case"
 fi

 # Legitimate URL forms pass the guard and reach git clone (stubbed, no network).
@@ -156,7 +156,7 @@ pass "service facades resolve enabled clones without widening replacement-bar ac
 require_compositor "plugin authentication boundary runtime test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping plugin authentication boundary runtime test"
+  skip "quickshell not installed; skipping plugin authentication boundary runtime test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "plugin registry contract test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping plugin registry contract test"
+  skip "quickshell not installed; skipping plugin registry contract test"
  exit 0
 fi

@@ -54,7 +54,7 @@ JS
 require_compositor "pointer movement gate runtime test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping pointer movement gate runtime test"
+  skip "quickshell not installed; skipping pointer movement gate runtime test"
  exit 0
 fi

@@ -20,7 +20,7 @@ trap cleanup EXIT
 require_compositor "shell runtime smoke test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping shell runtime smoke test"
+  skip "quickshell not installed; skipping shell runtime smoke test"
  exit 0
 fi

@@ -21,12 +21,12 @@ trap cleanup EXIT
 require_compositor "screenshot sanity test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping screenshot sanity test"
+  skip "quickshell not installed; skipping screenshot sanity test"
  exit 0
 fi

 if pgrep -x omasnap >/dev/null 2>&1; then
-  pass "omasnap is already running; skipping screenshot sanity test"
+  skip "omasnap is already running; skipping screenshot sanity test"
  exit 0
 fi

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"
+
+test_dir=$(mktemp -d)
+trap 'rm -rf "$test_dir"' EXIT
+mkdir -p "$test_dir/test/shell.d"
+cp "$ROOT/test/"{shell,all} "$test_dir/test/"
+cp "$SHELL_TEST_DIR/base-test.sh" "$test_dir/test/shell.d/"
+
+# Run the real runners in a tiny fixture tree, without recursing into this test.
+run_suite() {
+  status=0
+  output=$(bash "$test_dir/test/${1:-shell}" 2>&1) || status=$?
+}
+
+run_suite
+(( status == 1 )) && [[ $output == *"No shell tests found"* ]] ||
+  fail "an empty suite fails instead of counting the helper as a test" "$output"
+pass "an empty suite fails instead of counting the helper as a test"
+
+cat >"$test_dir/test/shell.d/a-pass-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+captured=$(skip "captured fixture output is not a skipped check")
+pass "the command skips an optional action"
+SH
+run_suite
+(( status == 0 )) && [[ $output == *"All 1 test files passed."* ]] ||
+  fail "passing assertions and captured skip output do not mark a file skipped" "$output"
+pass "passing assertions and captured skip output do not mark a file skipped"
+
+cat >"$test_dir/test/shell.d/b-partial-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+pass "static check"
+skip "first unavailable runtime check"
+skip "second unavailable runtime check"
+pass "check after skips"
+SH
+cat >"$test_dir/test/shell.d/c-skipped-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+unset WAYLAND_DISPLAY
+require_compositor "runtime fixture"
+fail "unreachable after compositor skip"
+SH
+cat >"$test_dir/test/shell.d/z-last-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+pass "last file ran"
+SH
+run_suite
+(( status == 0 )) && [[ $output == *"4 test files completed without failures; 2 had skipped checks."* ]] ||
+  fail "whole and partial skips are counted once per file and remain successful" "$output"
+[[ $output == *"ok - check after skips"* && $output == *"ok - last file ran"* ]] ||
+  fail "skip returns normally and later files still run" "$output"
+[[ $output == *$'Skipped checks in 2 of 4 test files:\n  test/shell.d/b-partial-test.sh\n  test/shell.d/c-skipped-test.sh\n'* ]] ||
+  fail "the summary identifies only files with skipped checks" "$output"
+pass "whole and partial skips are visible without failing or stopping the suite"
+
+cat >"$test_dir/test/shell.d/d-failed-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+skip "unavailable check before failure"
+echo "failure detail" >&2
+false
+pass "unreachable after failure"
+SH
+run_suite
+(( status == 1 )) && [[ $output == *$'1 of 5 test files failed:\n  test/shell.d/d-failed-test.sh'* ]] ||
+  fail "a failure after a skip still fails the suite through the output pipe" "$output"
+[[ $output == *"Skipped checks in 3 of 5 test files:"* && $output == *"failure detail"* && $output == *"ok - last file ran"* && $output != *"ok - unreachable after failure"* ]] ||
+  fail "failures preserve skip reporting, diagnostics, errexit, and later tests" "$output"
+pass "failures remain fatal while skips and later test results stay visible"
+
+printf '#!/bin/bash\necho "CLI fixture passed"\n' >"$test_dir/test/cli"
+chmod +x "$test_dir/test/cli"
+run_suite all
+(( status == 1 )) && [[ $output == *"CLI fixture passed"* && $output == *"Skipped checks in 3 of 5 test files:"* && $output == *$'1 of 2 suites failed:\n  test/shell'* ]] ||
+  fail "the aggregate runner preserves shell failures and skip reporting" "$output"
+pass "the aggregate runner preserves shell failures and skip reporting"
@@ -173,7 +173,7 @@ if locale -a 2>/dev/null | grep -qix 'en_US.utf-\?8'; then

  pass "the accepted set does not move with the desktop's locale"
 else
-  pass "no en_US.UTF-8 locale; skipping the locale-pinning check"
+  skip "no en_US.UTF-8 locale; skipping the locale-pinning check"
 fi

 # basename reads a leading dash as an option once the scp-style prefix is gone.
@@ -25,7 +25,7 @@ trap cleanup EXIT
 require_compositor "tray menu activation test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping tray menu activation test"
+  skip "quickshell not installed; skipping tray menu activation test"
  exit 0
 fi

@@ -36,7 +36,7 @@ python - <<'PY' || {
 import dbus
 import gi
 PY
-  pass "python DBus bindings unavailable; skipping tray menu activation test"
+  skip "python DBus bindings unavailable; skipping tray menu activation test"
  exit 0
 }

@@ -200,7 +200,7 @@ run_with_lock_env "$ROOT/bin/omarchy-update-stay-awake" stop
 pass "stale update ownership preserves a newer Stay Awake choice"

 # A stale PID is safe even if it has been reused by another process.
-sleep 30 &
+sleep 30 >/dev/null &
 unrelated_pid=$!
 unrelated_start_time=$(awk '{ print $22 }' "/proc/$unrelated_pid/stat")
 mkdir -p "$stay_awake_helper_state"
@@ -11,7 +11,7 @@ if [[ ${OMARCHY_WINDOWS_TEST_NAMESPACE:-0} != 1 ]]; then
    exec env OMARCHY_WINDOWS_TEST_NAMESPACE=1 \
      unshare --user --map-current-user --keep-caps --mount --propagation private bash "$0"
  fi
-  pass "unprivileged mount namespaces unavailable; skipping Windows VM mount runtime tests"
+  skip "unprivileged mount namespaces unavailable; skipping Windows VM mount runtime tests"
  exit 0
 fi

@@ -241,7 +241,7 @@ race_swaps="$TMPDIR/concurrent-race-swaps"
      sleep 0.005
    fi
  done
-) &
+) >/dev/null &
 racer_pid=$!
 concurrent_dc_calls=0
 dc() {
@@ -8,7 +8,7 @@ if ((EUID != 0)); then
  if unshare --user --map-auto --map-root-user --mount true 2>/dev/null; then
    exec unshare --user --map-auto --map-root-user --mount --propagation private bash "$0"
  fi
-  pass "automatic subordinate-id namespace unavailable; skipping root Windows VM boundary probe"
+  skip "automatic subordinate-id namespace unavailable; skipping root Windows VM boundary probe"
  exit 0
 fi