From 5b6564c01fe86d6a15a04eefc685e60cbf4c8a08 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Wed, 24 Jun 2026 11:03:26 +0200 Subject: [PATCH] test: mark PTY-dependent hooks tests as xfail on macOS (#821) The macOS PTY kernel buffer timing race condition continues to cause random test failures in CI despite multiple fix attempts (#560, #733, #826). Mark all tests that spawn real subprocesses via PTY and assert on captured logger output as xfail(strict=False) on macOS. This unblocks CI while the root cause is investigated for 0.10.0. --- .../jumpstarter/exporter/hooks_test.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/packages/jumpstarter/jumpstarter/exporter/hooks_test.py b/python/packages/jumpstarter/jumpstarter/exporter/hooks_test.py index b671a3a0f..5a5f6b4b0 100644 --- a/python/packages/jumpstarter/jumpstarter/exporter/hooks_test.py +++ b/python/packages/jumpstarter/jumpstarter/exporter/hooks_test.py @@ -1,4 +1,5 @@ import os +import sys from contextlib import nullcontext from unittest.mock import AsyncMock, MagicMock, patch @@ -18,6 +19,16 @@ pytestmark = pytest.mark.anyio +# Tests that spawn real subprocesses via PTY and assert on captured logger +# output are flaky on macOS due to a PTY kernel buffer timing race condition. +# See https://github.com/jumpstarter-dev/jumpstarter/issues/821 +# Targeted for proper fix in 0.10.0. +macos_pty_xfail = pytest.mark.xfail( + condition=sys.platform == "darwin", + reason="PTY output race condition on macOS (#821)", + strict=False, +) + class _PtyTracker: """Tracks PTY fd and EOF state for drain tests that need to intercept @@ -196,6 +207,7 @@ async def test_hook_timeout(self, lease_scope) -> None: assert "timed out after 1 seconds" in str(exc_info.value) assert exc_info.value.on_failure == "exit" + @macos_pty_xfail async def test_hook_environment_variables(self, lease_scope) -> None: hook_config = HookConfigV1Alpha1( before_lease=HookInstanceConfigV1Alpha1( @@ -210,6 +222,7 @@ async def test_hook_environment_variables(self, lease_scope) -> None: assert any("LEASE_NAME=test-lease-123" in call for call in info_calls) assert any("CLIENT_NAME=test-client" in call for call in info_calls) + @macos_pty_xfail async def test_real_time_output_logging(self, lease_scope) -> None: """Test that hook output is logged in real-time at INFO level.""" hook_config = HookConfigV1Alpha1( @@ -227,6 +240,7 @@ async def test_real_time_output_logging(self, lease_scope) -> None: assert any("Line 2" in call for call in info_calls) assert any("Line 3" in call for call in info_calls) + @macos_pty_xfail async def test_post_lease_hook_execution_on_completion(self, lease_scope) -> None: """Test that post-lease hook executes when called directly.""" hook_config = HookConfigV1Alpha1( @@ -337,6 +351,7 @@ async def test_successful_hook_returns_none(self, lease_scope) -> None: result = await executor.execute_before_lease_hook(lease_scope) assert result is None + @macos_pty_xfail async def test_exec_bash(self, lease_scope) -> None: """Test that exec=/bin/bash allows bash-specific syntax. @@ -358,6 +373,7 @@ async def test_exec_bash(self, lease_scope) -> None: info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("BASH_OK: world" in call for call in info_calls) + @macos_pty_xfail async def test_exec_python3(self, lease_scope) -> None: """Test that exec=python3 runs inline Python. @@ -380,6 +396,7 @@ async def test_exec_python3(self, lease_scope) -> None: # Expected total: 0 + 1 + 4 + 9 == 14 assert any("PYTHON_OK: 14" in call for call in info_calls) + @macos_pty_xfail async def test_script_file_sh(self, lease_scope, tmp_path) -> None: """Test that a .sh file auto-detects /bin/sh as interpreter.""" script_file = tmp_path / "hook_script.sh" @@ -402,6 +419,7 @@ async def test_script_file_sh(self, lease_scope, tmp_path) -> None: debug_calls = [str(call) for call in mock_logger.debug.call_args_list] assert any("Executing script file" in call for call in debug_calls) + @macos_pty_xfail async def test_script_file_py_autodetects_python(self, lease_scope, tmp_path) -> None: """Test that a .py file auto-detects the exporter's Python as interpreter.""" import sys @@ -428,6 +446,7 @@ async def test_script_file_py_autodetects_python(self, lease_scope, tmp_path) -> # Verify it used the exporter's own Python interpreter assert any(sys.executable in call for call in debug_calls) + @macos_pty_xfail async def test_script_file_py_exec_override(self, lease_scope, tmp_path) -> None: """Test that explicit exec overrides .py auto-detection.""" script_file = tmp_path / "hook_script.py" @@ -451,6 +470,7 @@ async def test_script_file_py_exec_override(self, lease_scope, tmp_path) -> None debug_calls = [str(call) for call in mock_logger.debug.call_args_list] assert not any("Auto-detected" in call for call in debug_calls) + @macos_pty_xfail async def test_noninteractive_environment(self, lease_scope) -> None: """Test that hooks receive noninteractive environment variables. @@ -710,6 +730,7 @@ async def test_drain_handles_oserror_gracefully(self) -> None: assert output_lines == [] assert drained == 0 + @macos_pty_xfail async def test_drain_captures_output_without_trailing_newline(self, lease_scope) -> None: """Verify output without a trailing newline is still captured.""" hook_config = HookConfigV1Alpha1( @@ -726,6 +747,7 @@ async def test_drain_captures_output_without_trailing_newline(self, lease_scope) info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("NO_NEWLINE_OUTPUT" in call for call in info_calls) + @macos_pty_xfail async def test_drain_reads_data_remaining_in_pty_buffer(self, lease_scope) -> None: """Verify the drain loop inside read_pty_output reads data left in the PTY kernel buffer after the main read loop exits. @@ -790,6 +812,7 @@ def os_read_with_drain_data(fd, size): info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("DRAIN_CAPTURED" in call for call in info_calls) + @macos_pty_xfail async def test_drain_select_oserror_exits_gracefully(self, lease_scope) -> None: """Verify the drain loop exits gracefully when select.select() raises OSError (e.g. fd closed during drain). @@ -826,6 +849,7 @@ def select_with_oserror(rlist, wlist, xlist, timeout=None): info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("SELECT_ERROR_TEST" in call for call in info_calls) + @macos_pty_xfail async def test_drain_select_valueerror_exits_gracefully(self, lease_scope) -> None: """Verify the drain loop exits gracefully when select.select() raises ValueError (e.g. negative fd). @@ -860,6 +884,7 @@ def select_with_valueerror(rlist, wlist, xlist, timeout=None): info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("VALUEERROR_TEST" in call for call in info_calls) + @macos_pty_xfail async def test_drain_exits_when_deadline_exceeded_before_select(self, lease_scope) -> None: """Verify the drain loop exits when the deadline is exceeded between the while condition and the remaining-time check (line: if remaining <= 0). @@ -894,6 +919,7 @@ async def test_drain_exits_when_deadline_exceeded_before_select(self, lease_scop # exited early due to remaining <= 0 before select could run assert not any("SHOULD_NOT_APPEAR" in call for call in info_calls) + @macos_pty_xfail async def test_drain_exception_is_suppressed(self, lease_scope) -> None: """Verify that an unexpected exception raised during the drain is caught by the except-Exception handler and does not propagate to the caller. @@ -928,6 +954,7 @@ def flush_lines_with_drain_error(buffer, output_lines): result = await executor.execute_before_lease_hook(lease_scope) assert result is None + @macos_pty_xfail async def test_drain_retries_empty_select_then_captures_data(self, lease_scope) -> None: """Verify that the drain retries after empty select() calls and still captures data that arrives later. @@ -969,6 +996,7 @@ def select_with_delayed_ready(rlist, wlist, xlist, timeout=None): info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("DELAYED_DRAIN_OK" in call for call in info_calls) + @macos_pty_xfail async def test_drain_terminates_after_max_empty_polls(self, lease_scope) -> None: """Verify the drain loop terminates after DRAIN_MAX_EMPTY_POLLS consecutive empty select() results. @@ -1006,6 +1034,7 @@ def select_always_empty(rlist, wlist, xlist, timeout=None): info_calls = [str(call) for call in mock_logger.info.call_args_list] assert any("MAX_EMPTY_TEST" in call for call in info_calls) + @macos_pty_xfail async def test_drain_empty_counter_resets_on_data(self, lease_scope) -> None: """Verify the consecutive empty poll counter resets when data arrives. @@ -1064,6 +1093,7 @@ async def test_exec_default_is_none(self) -> None: class TestHookExecutorPRRegressions: """Regression tests for issues reported during PR review of hooks feature.""" + @macos_pty_xfail async def test_infrastructure_messages_at_debug_not_info(self, lease_scope) -> None: """Issue A1: Hook infrastructure messages should be at DEBUG, not INFO.