Report skipped checks separately in shell tests

2026-09-21 11:49:32 +02:00
1 parent 961ec7f39f
commit 599a6a665b
@@ -52,6 +52,7 @@ directory or an installed Omarchy.
 Assertions are TAP-flavored and blunt:

 - `pass "description"` prints `ok - description`.
+- `skip "description"` prints `ok - description # SKIP`. Include why the check could not run. Like `pass`, it returns normally; use `exit 0` afterwards only when the rest of the file cannot run either.
 - `fail "description" [detail]` prints the optional detail and
  `not ok - description` to stderr, then **exits the file**. There is no
  counting or continuing within a file: the first failed assertion ends it,
@@ -59,19 +60,13 @@ Assertions are TAP-flavored and blunt:
  already invalidated.
 - `require_command <cmd>` fails the file when a needed tool is absent.

-The runner compensates for that early exit: `./test/shell` continues past a
-failing file and summarizes the failures at the end. Aborting the whole run at
-the first bad file once let a single packaging failure mask 114 of 134 files.
-Failure granularity is therefore per file inside a run, per assertion inside a
-file.
+The runner compensates for that early exit: `./test/shell` continues past a failing file and summarizes the failures at the end. Aborting the whole run at the first bad file once let a single packaging failure mask 114 of 134 files. Failure granularity is therefore per file inside a run, per assertion inside a file.
+
+The runner also lists files with skipped checks, including files that ran some checks or later failed. Skips do not fail a run. When no files fail, a run with skips is reported as completed without failures rather than having passed every check.

 ## Compositor-dependent tests

-Some tests launch Quickshell or query Hyprland, but the suite must stay green
-on headless machines. `require_compositor "description"` handles this: when no
-compositor answers it prints `ok - no Wayland compositor; skipping ...` and
-exits 0 — a skip is a passing test — and otherwise returns so the file
-proceeds.
+Some tests launch Quickshell or query Hyprland, but the suite must stay green on headless machines. `require_compositor "description"` handles this: when no compositor answers it calls `skip` with the reason and exits 0, and otherwise returns so the file proceeds. The skip marker lets the runner distinguish unavailable runtime coverage from checks that passed.

 The probe is more than an environment check, because `WAYLAND_DISPLAY` only
 proves the variable was inherited. Sandboxes pass the environment through
@@ -119,6 +114,7 @@ only a live session can prove.

 ## Conventions worth copying

+- **Redirect background output.** Send background fixtures' stdout to a log or `/dev/null` and clean up the processes on exit. An inherited output pipe can hold the runner open after the test exits.
 - **Stub the world, run the real code.** Tests build a scratch `bin/` of stub
  executables (`sudo`, `tmux`, `gsettings`, helper commands) that log their
  arguments to a file, prepend it to `PATH`, and then run the real script
@@ -21,16 +21,31 @@ fi
 # Keep going after a failing file. A test file exits at its first failed
 # assertion, so aborting the run there too would hide every file after it --
 # one packaging failure was masking 114 of 134 files.
+output=$(mktemp)
+trap 'rm -f "$output"' EXIT
 failed=()
+skipped=()
 for test in "${tests[@]}"; do
  printf '==> %s\n' "${test#$ROOT/}"
-  bash "$test" || failed+=("${test#$ROOT/}")
+  bash "$test" | tee "$output" || failed+=("${test#$ROOT/}")
+  if grep -q '^ok - .* # SKIP$' "$output"; then
+    skipped+=("${test#$ROOT/}")
+  fi
 done

+if (( ${#skipped[@]} > 0 )); then
+  printf '\nSkipped checks in %d of %d test files:\n' "${#skipped[@]}" "${#tests[@]}"
+  printf '  %s\n' "${skipped[@]}"
+fi
+
 if (( ${#failed[@]} > 0 )); then
  printf '\n%d of %d test files failed:\n' "${#failed[@]}" "${#tests[@]}" >&2
  printf '  %s\n' "${failed[@]}" >&2
  exit 1
 fi

-printf '\nAll %d test files passed.\n' "${#tests[@]}"
+if (( ${#skipped[@]} == 0 )); then
+  printf '\nAll %d test files passed.\n' "${#tests[@]}"
+else
+  printf '\n%d test files completed without failures; %d had skipped checks.\n' "${#tests[@]}" "${#skipped[@]}"
+fi
@@ -43,7 +43,7 @@ if (( EUID != 0 )); then
 fi

 if (( ! root_runtime_available )); then
-  pass "no unprivileged user namespace; skipping the root lock-helper lookup matrix"
+  skip "no unprivileged user namespace; skipping the root lock-helper lookup matrix"
  exit 0
 fi

@@ -7,7 +7,7 @@ source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"
 require_compositor "bar icon geometry test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping bar icon geometry test"
+  skip "quickshell not installed; skipping bar icon geometry test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "bar widget contract test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping bar widget contract test"
+  skip "quickshell not installed; skipping bar widget contract test"
  exit 0
 fi

@@ -14,6 +14,10 @@ pass() {
  printf 'ok - %s\n' "$1"
 }

+skip() {
+  printf 'ok - %s # SKIP\n' "$1"
+}
+
 fail() {
  local description="$1"
  local detail="${2:-}"
@@ -72,7 +76,7 @@ require_compositor() {
    return 0
  fi

-  pass "no Wayland compositor; skipping $description"
+  skip "no Wayland compositor; skipping $description"
  exit 0
 }

@@ -123,7 +123,7 @@ if [[ -n $real_hiddev ]]; then
    fail "wrapper did not trust a valid cached hiddev node: $real_hiddev"
  pass "wrapper trusts a cached hiddev character device without re-detecting"
 else
-  pass "no /dev/hiddev* character device present; skipping the valid-cache case"
+  skip "no /dev/hiddev* character device present; skipping the valid-cache case"
 fi

 # --- With no XDG_RUNTIME_DIR, the predictable /tmp cache is not consulted ------
@@ -148,5 +148,5 @@ if mkfifo "$tmp_cache" 2>/dev/null; then
      "it blocked reading the FIFO decoy at $tmp_cache"
  pass "wrapper ignores the /tmp cache path when XDG_RUNTIME_DIR is unset"
 else
-  pass "$tmp_cache already present or not safely creatable; skipping the /tmp-fallback case"
+  skip "$tmp_cache already present or not safely creatable; skipping the /tmp-fallback case"
 fi
@@ -46,7 +46,7 @@ mode=$(stat -c '%a' "$write_dir/color.json")
 pass "theme colour writes a 0644 color.json"

 if (( EUID == 0 )); then
-  pass "running as root; skipping the mktemp-failure check"
+  skip "running as root; skipping the mktemp-failure check"
 else
  chmod u+w "$write_dir"
  export TMPDIR=$test_tmp/missing-tmp
@@ -82,7 +82,7 @@ SH
 chmod +x "$stub_bin/sudo"

 if ((EUID == 0)); then
-  pass "running as root; skipping the elevation checks, which would rewrite this machine's browser policy"
+  skip "running as root; skipping the elevation checks, which would rewrite this machine's browser policy"
 else
  elevation_for() {
    : >"$test_tmp/elevation"
@@ -20,7 +20,7 @@ JS
 require_compositor "Button hover geometry runtime test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping Button hover geometry runtime test"
+  skip "quickshell not installed; skipping Button hover geometry runtime test"
  exit 0
 fi

@@ -6,7 +6,7 @@ source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"

 # Exercise the privileged installer without writing to the host's /usr/share.
 if ! command -v bwrap >/dev/null || ! bwrap --ro-bind / / --unshare-user --uid 0 --gid 0 true 2>/dev/null; then
-  pass "user namespaces unavailable; skipping isolated Claude extension installation"
+  skip "user namespaces unavailable; skipping isolated Claude extension installation"
  exit 0
 fi

@@ -62,7 +62,7 @@ run_guard() {
  ' bash "$SHELL_TEST_DIR" 2>&1 || printf 'guard exited %s\n' "$?"
 }

-skipped="ok - no Wayland compositor; skipping sample runtime test"
+skipped="ok - no Wayland compositor; skipping sample runtime test # SKIP"

 output=$(run_guard -u WAYLAND_DISPLAY XDG_RUNTIME_DIR="$runtime_dir")
 [[ $output == "$skipped" ]] || fail "guard skips without a display" "$output"
@@ -76,7 +76,7 @@ pass "guard skips when the socket is unreachable"

 # Everything below needs a socket to stand in for a live or abandoned compositor.
 if (( ! socket_bound )); then
-  pass "cannot bind a Unix socket here; skipping the cases that need one"
+  skip "cannot bind a Unix socket here; skipping the cases that need one"
  exit 0
 fi

@@ -81,13 +81,13 @@ SH
  rm -rf "$poison_dir"
  pass "root omarchy-dns resolves system helpers from a trusted PATH, not the invocation PATH"
 else
-  pass "no unprivileged user namespace; skipping the root trusted-PATH probe"
+  skip "no unprivileged user namespace; skipping the root trusted-PATH probe"
 fi

 # require_root returns immediately for root, so the stubs below would not stand
 # between the script and the host's real NetworkManager and resolved config.
 if (( EUID == 0 )); then
-  pass "running as root; skipping the elevation checks, which would rewrite this machine's DNS"
+  skip "running as root; skipping the elevation checks, which would rewrite this machine's DNS"
  exit 0
 fi

@@ -8,7 +8,7 @@ if (( EUID != 0 )); then
  if unshare --user --map-root-user true 2>/dev/null; then
    exec unshare --user --map-root-user bash "$0"
  fi
-  pass "no unprivileged user namespace; skipping factory account cleanup"
+  skip "no unprivileged user namespace; skipping factory account cleanup"
  exit 0
 fi

@@ -4,7 +4,7 @@ source "$(dirname "$0")/base-test.sh"

 # The hook guards on the real /etc/pam.d path, which can't be mocked via PATH.
 if [[ -f /etc/pam.d/omarchy-lock-fingerprint ]]; then
-  pass "fingerprint invitation test skipped: host already has fingerprint auth configured"
+  skip "fingerprint invitation test skipped: host already has fingerprint auth configured"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "QML contract test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping QML contract test"
+  skip "quickshell not installed; skipping QML contract test"
  exit 0
 fi

@@ -150,5 +150,5 @@ pass "font controls create missing Kitty overrides without restoring the theme i
 if "$ROOT/bin/omarchy-cmd-present" kitty; then
  kitty +runpy "$(cat "$ROOT/test/shell.d/fixtures/kitty/check-config.py")"
 else
-  pass "Kitty not installed; skipping native config parser checks"
+  skip "Kitty not installed; skipping native config parser checks"
 fi
@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "lock fingerprint indicator test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping lock fingerprint indicator test"
+  skip "quickshell not installed; skipping lock fingerprint indicator test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "lock password overflow test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping lock password overflow test"
+  skip "quickshell not installed; skipping lock password overflow test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "manifest entrypoint load test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping manifest entrypoint load test"
+  skip "quickshell not installed; skipping manifest entrypoint load test"
  exit 0
 fi

@@ -3,7 +3,7 @@ set -euo pipefail
 source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell unavailable; skipping lock feed QML lifecycle"
+  skip "quickshell unavailable; skipping lock feed QML lifecycle"
  exit 0
 fi
 work=$(mktemp -d)
@@ -158,7 +158,7 @@ if script -qec true /dev/null >/dev/null 2>&1; then
    fail "plugin add reached git clone for an option-shaped gum-prompt URL"
  pass "plugin add guard rejects an option-shaped URL from the interactive prompt"
 else
-  pass "script -qec unavailable; skipping the interactive gum-prompt guard case"
+  skip "script -qec unavailable; skipping the interactive gum-prompt guard case"
 fi

 # Legitimate URL forms pass the guard and reach git clone (stubbed, no network).
@@ -156,7 +156,7 @@ pass "service facades resolve enabled clones without widening replacement-bar ac
 require_compositor "plugin authentication boundary runtime test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping plugin authentication boundary runtime test"
+  skip "quickshell not installed; skipping plugin authentication boundary runtime test"
  exit 0
 fi

@@ -21,7 +21,7 @@ trap cleanup EXIT
 require_compositor "plugin registry contract test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping plugin registry contract test"
+  skip "quickshell not installed; skipping plugin registry contract test"
  exit 0
 fi

@@ -54,7 +54,7 @@ JS
 require_compositor "pointer movement gate runtime test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping pointer movement gate runtime test"
+  skip "quickshell not installed; skipping pointer movement gate runtime test"
  exit 0
 fi

@@ -20,7 +20,7 @@ trap cleanup EXIT
 require_compositor "shell runtime smoke test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping shell runtime smoke test"
+  skip "quickshell not installed; skipping shell runtime smoke test"
  exit 0
 fi

@@ -21,12 +21,12 @@ trap cleanup EXIT
 require_compositor "screenshot sanity test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping screenshot sanity test"
+  skip "quickshell not installed; skipping screenshot sanity test"
  exit 0
 fi

 if pgrep -x omasnap >/dev/null 2>&1; then
-  pass "omasnap is already running; skipping screenshot sanity test"
+  skip "omasnap is already running; skipping screenshot sanity test"
  exit 0
 fi

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/base-test.sh"
+
+test_dir=$(mktemp -d)
+trap 'rm -rf "$test_dir"' EXIT
+mkdir -p "$test_dir/test/shell.d"
+cp "$ROOT/test/"{shell,all} "$test_dir/test/"
+cp "$SHELL_TEST_DIR/base-test.sh" "$test_dir/test/shell.d/"
+
+# Run the real runners in a tiny fixture tree, without recursing into this test.
+run_suite() {
+  status=0
+  output=$(bash "$test_dir/test/${1:-shell}" 2>&1) || status=$?
+}
+
+run_suite
+(( status == 1 )) && [[ $output == *"No shell tests found"* ]] ||
+  fail "an empty suite fails instead of counting the helper as a test" "$output"
+pass "an empty suite fails instead of counting the helper as a test"
+
+cat >"$test_dir/test/shell.d/a-pass-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+captured=$(skip "captured fixture output is not a skipped check")
+pass "the command skips an optional action"
+SH
+run_suite
+(( status == 0 )) && [[ $output == *"All 1 test files passed."* ]] ||
+  fail "passing assertions and captured skip output do not mark a file skipped" "$output"
+pass "passing assertions and captured skip output do not mark a file skipped"
+
+cat >"$test_dir/test/shell.d/b-partial-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+pass "static check"
+skip "first unavailable runtime check"
+skip "second unavailable runtime check"
+pass "check after skips"
+SH
+cat >"$test_dir/test/shell.d/c-skipped-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+unset WAYLAND_DISPLAY
+require_compositor "runtime fixture"
+fail "unreachable after compositor skip"
+SH
+cat >"$test_dir/test/shell.d/z-last-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+pass "last file ran"
+SH
+run_suite
+(( status == 0 )) && [[ $output == *"4 test files completed without failures; 2 had skipped checks."* ]] ||
+  fail "whole and partial skips are counted once per file and remain successful" "$output"
+[[ $output == *"ok - check after skips"* && $output == *"ok - last file ran"* ]] ||
+  fail "skip returns normally and later files still run" "$output"
+[[ $output == *$'Skipped checks in 2 of 4 test files:\n  test/shell.d/b-partial-test.sh\n  test/shell.d/c-skipped-test.sh\n'* ]] ||
+  fail "the summary identifies only files with skipped checks" "$output"
+pass "whole and partial skips are visible without failing or stopping the suite"
+
+cat >"$test_dir/test/shell.d/d-failed-test.sh" <<'SH'
+set -euo pipefail
+source "$(dirname "$0")/base-test.sh"
+skip "unavailable check before failure"
+echo "failure detail" >&2
+false
+pass "unreachable after failure"
+SH
+run_suite
+(( status == 1 )) && [[ $output == *$'1 of 5 test files failed:\n  test/shell.d/d-failed-test.sh'* ]] ||
+  fail "a failure after a skip still fails the suite through the output pipe" "$output"
+[[ $output == *"Skipped checks in 3 of 5 test files:"* && $output == *"failure detail"* && $output == *"ok - last file ran"* && $output != *"ok - unreachable after failure"* ]] ||
+  fail "failures preserve skip reporting, diagnostics, errexit, and later tests" "$output"
+pass "failures remain fatal while skips and later test results stay visible"
+
+printf '#!/bin/bash\necho "CLI fixture passed"\n' >"$test_dir/test/cli"
+chmod +x "$test_dir/test/cli"
+run_suite all
+(( status == 1 )) && [[ $output == *"CLI fixture passed"* && $output == *"Skipped checks in 3 of 5 test files:"* && $output == *$'1 of 2 suites failed:\n  test/shell'* ]] ||
+  fail "the aggregate runner preserves shell failures and skip reporting" "$output"
+pass "the aggregate runner preserves shell failures and skip reporting"
@@ -173,7 +173,7 @@ if locale -a 2>/dev/null | grep -qix 'en_US.utf-\?8'; then

  pass "the accepted set does not move with the desktop's locale"
 else
-  pass "no en_US.UTF-8 locale; skipping the locale-pinning check"
+  skip "no en_US.UTF-8 locale; skipping the locale-pinning check"
 fi

 # basename reads a leading dash as an option once the scp-style prefix is gone.
@@ -25,7 +25,7 @@ trap cleanup EXIT
 require_compositor "tray menu activation test"

 if ! command -v quickshell >/dev/null 2>&1; then
-  pass "quickshell not installed; skipping tray menu activation test"
+  skip "quickshell not installed; skipping tray menu activation test"
  exit 0
 fi

@@ -36,7 +36,7 @@ python - <<'PY' || {
 import dbus
 import gi
 PY
-  pass "python DBus bindings unavailable; skipping tray menu activation test"
+  skip "python DBus bindings unavailable; skipping tray menu activation test"
  exit 0
 }

@@ -200,7 +200,7 @@ run_with_lock_env "$ROOT/bin/omarchy-update-stay-awake" stop
 pass "stale update ownership preserves a newer Stay Awake choice"

 # A stale PID is safe even if it has been reused by another process.
-sleep 30 &
+sleep 30 >/dev/null &
 unrelated_pid=$!
 unrelated_start_time=$(awk '{ print $22 }' "/proc/$unrelated_pid/stat")
 mkdir -p "$stay_awake_helper_state"
@@ -11,7 +11,7 @@ if [[ ${OMARCHY_WINDOWS_TEST_NAMESPACE:-0} != 1 ]]; then
    exec env OMARCHY_WINDOWS_TEST_NAMESPACE=1 \
      unshare --user --map-current-user --keep-caps --mount --propagation private bash "$0"
  fi
-  pass "unprivileged mount namespaces unavailable; skipping Windows VM mount runtime tests"
+  skip "unprivileged mount namespaces unavailable; skipping Windows VM mount runtime tests"
  exit 0
 fi

@@ -241,7 +241,7 @@ race_swaps="$TMPDIR/concurrent-race-swaps"
      sleep 0.005
    fi
  done
-) &
+) >/dev/null &
 racer_pid=$!
 concurrent_dc_calls=0
 dc() {
@@ -8,7 +8,7 @@ if ((EUID != 0)); then
  if unshare --user --map-auto --map-root-user --mount true 2>/dev/null; then
    exec unshare --user --map-auto --map-root-user --mount --propagation private bash "$0"
  fi
-  pass "automatic subordinate-id namespace unavailable; skipping root Windows VM boundary probe"
+  skip "automatic subordinate-id namespace unavailable; skipping root Windows VM boundary probe"
  exit 0
 fi