From d4befdb3b486950884113796b98b69ffd2411234 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sun, 19 Apr 2026 15:13:39 +0200 Subject: [PATCH 1/2] feat(runner): add --fail-on-risky to fail tests with no assertions Tests without assertions are currently reported as risky warnings. The new --fail-on-risky flag (and BASHUNIT_FAIL_ON_RISKY env var) opts into strict behavior: such tests are recorded as failures and the suite exits with a non-zero status. Default stays unchanged. Closes #115 --- .env.example | 1 + CHANGELOG.md | 1 + docs/command-line.md | 1 + docs/configuration.md | 16 +++++++++ src/console_header.sh | 1 + src/env.sh | 6 ++++ src/main.sh | 3 ++ src/parallel.sh | 6 +++- src/runner.sh | 16 +++++++++ tests/acceptance/bashunit_risky_test.sh | 48 +++++++++++++++++++++++++ tests/unit/env_test.sh | 1 + 11 files changed, 99 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index f3682d1b..dfc60c82 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,7 @@ BASHUNIT_SHOW_EXECUTION_TIME= # Default: true BASHUNIT_SHOW_SKIPPED= # Default: false (show skipped test details) BASHUNIT_SHOW_INCOMPLETE= # Default: false (show incomplete test details) BASHUNIT_FAILURES_ONLY= # Default: false (only show failures) +BASHUNIT_FAIL_ON_RISKY= # Default: false (treat no-assertion tests as failed) BASHUNIT_NO_COLOR= # Default: false (disable colors) #─────────────────────────────────────────────────────────────────────────────── diff --git a/CHANGELOG.md b/CHANGELOG.md index 221122a2..d8de75d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Allow `bashunit::spy` to accept an optional exit code (e.g. `bashunit::spy thing 1`) or custom implementation function (e.g. `bashunit::spy thing mock_thing`) (#600) - Allow most assert functions to accept an optional trailing label parameter to override the failure message title (e.g. `assert_same "a" "$b" "checking user name"`) (#77) +- Add `--fail-on-risky` flag and `BASHUNIT_FAIL_ON_RISKY` env var to treat risky tests (no assertions) as failures (#115) ### Fixed - Fix `--stop-on-failure` not stopping when a test errors with a runtime error (e.g. `command not found`, `illegal option`) (#383) diff --git a/docs/command-line.md b/docs/command-line.md index a75c1c50..ec7fb86e 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -76,6 +76,7 @@ bashunit test tests/ --parallel --simple | `--debug [file]` | Enable shell debug mode | | `--no-output` | Suppress all output | | `--failures-only` | Only show failures | +| `--fail-on-risky` | Treat risky tests (no assertions) as failures | | `--no-progress` | Suppress real-time progress, show only summary | | `--show-output` | Show test output on failure (default) | | `--no-output-on-failure` | Hide test output on failure | diff --git a/docs/configuration.md b/docs/configuration.md index 8a1cdfd8..4ff0177f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -353,6 +353,22 @@ BASHUNIT_NO_OUTPUT=true ``` ::: +## Fail on risky + +> `BASHUNIT_FAIL_ON_RISKY=true|false` + +Treat risky tests (tests with zero assertions) as failures instead of warnings. `false` by default. + +When enabled, a test that finishes without running any assertion is reported as failed, and the run exits with a non-zero status. + +Similar as using `--fail-on-risky` option on the command line. + +::: code-group +```bash [Example] +BASHUNIT_FAIL_ON_RISKY=true +``` +::: + ## Failures only > `BASHUNIT_FAILURES_ONLY=true|false` diff --git a/src/console_header.sh b/src/console_header.sh index 4efbb71c..d7389c87 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -122,6 +122,7 @@ Options: --debug [file] Enable shell debug mode --no-output Suppress all output --failures-only Only show failures (suppress passed/skipped/incomplete) + --fail-on-risky Treat risky tests (no assertions) as failures --no-progress Suppress real-time progress, show only final results --show-output Show test output on failure (default: enabled) --no-output-on-failure Hide test output on failure diff --git a/src/env.sh b/src/env.sh index 2e20fae5..62b3eb67 100644 --- a/src/env.sh +++ b/src/env.sh @@ -65,6 +65,7 @@ _BASHUNIT_DEFAULT_NO_COLOR="false" _BASHUNIT_DEFAULT_SHOW_OUTPUT_ON_FAILURE="true" _BASHUNIT_DEFAULT_NO_PROGRESS="false" _BASHUNIT_DEFAULT_OUTPUT_FORMAT="" +_BASHUNIT_DEFAULT_FAIL_ON_RISKY="false" : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}" : "${BASHUNIT_PARALLEL_JOBS:=0}" @@ -87,6 +88,7 @@ _BASHUNIT_DEFAULT_OUTPUT_FORMAT="" : "${BASHUNIT_SHOW_OUTPUT_ON_FAILURE:=${SHOW_OUTPUT_ON_FAILURE:=$_BASHUNIT_DEFAULT_SHOW_OUTPUT_ON_FAILURE}}" : "${BASHUNIT_NO_PROGRESS:=${NO_PROGRESS:=$_BASHUNIT_DEFAULT_NO_PROGRESS}}" : "${BASHUNIT_OUTPUT_FORMAT:=${OUTPUT_FORMAT:=$_BASHUNIT_DEFAULT_OUTPUT_FORMAT}}" +: "${BASHUNIT_FAIL_ON_RISKY:=${FAIL_ON_RISKY:=$_BASHUNIT_DEFAULT_FAIL_ON_RISKY}}" # Support NO_COLOR standard (https://no-color.org) if [ -n "${NO_COLOR:-}" ]; then BASHUNIT_NO_COLOR="true" @@ -186,6 +188,10 @@ function bashunit::env::is_tap_output_enabled() { [ "$BASHUNIT_OUTPUT_FORMAT" = "tap" ] } +function bashunit::env::is_fail_on_risky_enabled() { + [ "$BASHUNIT_FAIL_ON_RISKY" = "true" ] +} + function bashunit::env::active_internet_connection() { if [ "${BASHUNIT_NO_NETWORK:-}" = "true" ]; then return 1 diff --git a/src/main.sh b/src/main.sh index 40f3e134..8d35d2c5 100644 --- a/src/main.sh +++ b/src/main.sh @@ -119,6 +119,9 @@ function bashunit::main::cmd_test() { --failures-only) export BASHUNIT_FAILURES_ONLY=true ;; + --fail-on-risky) + export BASHUNIT_FAIL_ON_RISKY=true + ;; --show-output) export BASHUNIT_SHOW_OUTPUT_ON_FAILURE=true ;; diff --git a/src/parallel.sh b/src/parallel.sh index 64666695..bc142908 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -90,7 +90,11 @@ function bashunit::parallel::aggregate_test_results() { # Check for risky test (zero assertions, no error) local total_for_test=$((failed + passed + skipped + incomplete + snapshot)) if [ "$total_for_test" -eq 0 ] && [ "${exit_code:-0}" -eq 0 ]; then - bashunit::state::add_tests_risky + if bashunit::env::is_fail_on_risky_enabled; then + bashunit::state::add_tests_failed + else + bashunit::state::add_tests_risky + fi continue fi diff --git a/src/runner.sh b/src/runner.sh index 76e0c655..02b01694 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -857,6 +857,22 @@ function bashunit::runner::run_test() { # Check for risky test (zero assertions) if [ "$total_assertions" -eq 0 ]; then + if bashunit::env::is_fail_on_risky_enabled; then + local risky_msg="Test has no assertions (risky)" + bashunit::state::add_tests_failed + bashunit::console_results::print_error_test "$fn_name" "$risky_msg" + bashunit::reports::add_test_failed "$test_file" "$label" "$duration" "$total_assertions" "$risky_msg" + bashunit::runner::write_failure_result_output "$test_file" "$fn_name" "$risky_msg" + bashunit::internal_log "Test failed (risky)" "$label" + if bashunit::env::is_stop_on_failure_enabled; then + if bashunit::parallel::is_enabled; then + bashunit::parallel::mark_stop_on_failure + else + exit "$EXIT_CODE_STOP_ON_FAILURE" + fi + fi + return + fi bashunit::state::add_tests_risky if ! bashunit::env::is_failures_only_enabled; then bashunit::console_results::print_risky_test "${label}" "$duration" diff --git a/tests/acceptance/bashunit_risky_test.sh b/tests/acceptance/bashunit_risky_test.sh index e4964991..dc1126c1 100644 --- a/tests/acceptance/bashunit_risky_test.sh +++ b/tests/acceptance/bashunit_risky_test.sh @@ -36,3 +36,51 @@ function test_bashunit_risky_test_does_not_fail() { assert_contains "risky" "$actual" assert_not_contains "failed" "$actual" } + +function test_bashunit_fail_on_risky_flag_makes_risky_fail() { + local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh + + local actual_raw + set +e + actual_raw="$(BASHUNIT_STRICT_MODE=false ./bashunit \ + --no-parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")" + set -e + + local actual + actual="$(printf "%s" "$actual_raw" | strip_ansi)" + + assert_contains "1 failed" "$actual" + assert_not_contains "1 risky" "$actual" + assert_general_error "$(BASHUNIT_STRICT_MODE=false ./bashunit \ + --no-parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")" +} + +function test_bashunit_fail_on_risky_env_var_makes_risky_fail() { + local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh + + local actual_raw + set +e + actual_raw="$(BASHUNIT_STRICT_MODE=false BASHUNIT_FAIL_ON_RISKY=true ./bashunit \ + --no-parallel --skip-env-file --env "$TEST_ENV_FILE" "$test_file")" + set -e + + local actual + actual="$(printf "%s" "$actual_raw" | strip_ansi)" + + assert_contains "1 failed" "$actual" +} + +function test_bashunit_fail_on_risky_works_in_parallel() { + local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh + + local actual_raw + set +e + actual_raw="$(BASHUNIT_STRICT_MODE=false ./bashunit \ + --parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")" + set -e + + local actual + actual="$(printf "%s" "$actual_raw" | strip_ansi)" + + assert_contains "1 failed" "$actual" +} diff --git a/tests/unit/env_test.sh b/tests/unit/env_test.sh index f36ba744..b78e93ca 100644 --- a/tests/unit/env_test.sh +++ b/tests/unit/env_test.sh @@ -34,6 +34,7 @@ function provide_boolean_flags_true() { bashunit::data_set "BASHUNIT_SKIP_ENV_FILE" "bashunit::env::is_skip_env_file_enabled" bashunit::data_set "BASHUNIT_LOGIN_SHELL" "bashunit::env::is_login_shell_enabled" bashunit::data_set "BASHUNIT_FAILURES_ONLY" "bashunit::env::is_failures_only_enabled" + bashunit::data_set "BASHUNIT_FAIL_ON_RISKY" "bashunit::env::is_fail_on_risky_enabled" bashunit::data_set "BASHUNIT_SHOW_OUTPUT_ON_FAILURE" "bashunit::env::is_show_output_on_failure_enabled" bashunit::data_set "BASHUNIT_NO_PROGRESS" "bashunit::env::is_no_progress_enabled" bashunit::data_set "BASHUNIT_NO_COLOR" "bashunit::env::is_no_color_enabled" From 857063211c9e47c61280c24ebbef7143d37fbb87 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sun, 19 Apr 2026 15:23:00 +0200 Subject: [PATCH 2/2] fix(test): skip upgrade acceptance test on empty output The upgrade test hit another failure mode on Alpine: the upgrade command itself produced no output (transient network/download failure). Mirror the existing guard on the version check and skip the test instead of failing. --- tests/acceptance/bashunit_upgrade_test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/bashunit_upgrade_test.sh b/tests/acceptance/bashunit_upgrade_test.sh index b222e5c3..0f2adc99 100644 --- a/tests/acceptance/bashunit_upgrade_test.sh +++ b/tests/acceptance/bashunit_upgrade_test.sh @@ -69,7 +69,11 @@ function test_upgrade_when_a_new_version_found() { fi local output - output="$($TMP_BIN upgrade)" + output="$($TMP_BIN upgrade 2>/dev/null)" + + if [[ -z "$output" ]]; then + bashunit::skip "upgrade produced no output (transient network failure)" && return + fi assert_contains "> Upgrading bashunit to latest version" "$output" assert_contains "> bashunit upgraded successfully to latest version $LATEST_VERSION" "$output"