Add suppress_run_tests() to prevent double test execution

When submission_tests scripts are loaded as modules by task.py,
any rf.run_tests() call at module level would execute tests during
import. Adding a suppression mechanism allows task.py to prevent
this, with a deprecation warning nudging authors to remove the
manual rf.run_tests() call.

Author: Nils Bars

ref_utils/__init__.py

@@ -25,6 +25,7 @@
     "add_environment_test",
     "add_submission_test",
     "run_tests",
+    "suppress_run_tests",
     # Process
     "drop_privileges",
     "run",
@@ -57,6 +58,7 @@
     environment_test,
     run_tests,
     submission_test,
+    suppress_run_tests,
 )
 from .process import (
     drop_privileges,

ref_utils/decorator.py

@@ -11,6 +11,7 @@
 
 DEFAULT_TASK_NAME = "default"
 __registered_tasks: ty.Dict[str, "_Task"] = {}
+_suppress_run_tests: bool = False
 
 
 @dataclass
@@ -124,6 +125,12 @@ def wrapper(*args: str, **kwargs: Any) -> Any:
     return _extended_submission_test
 
 
+def suppress_run_tests(suppress: bool = True) -> None:
+    """Suppress run_tests() calls (used by task.py to prevent double execution)."""
+    global _suppress_run_tests
+    _suppress_run_tests = suppress
+
+
 def run_tests(
     *,
     result_will_be_submitted: bool = False,
@@ -139,6 +146,16 @@ def run_tests(
     Returns:
         A list of TaskTestResult objects containing the results of each task.
     """
+    if _suppress_run_tests:
+        warnings.warn(
+            "Calling rf.run_tests() at the end of submission_tests is deprecated "
+            "and will be removed in a future version. "
+            "Remove the rf.run_tests() call; tests are executed automatically.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return []
+
     # Set env var so test_result_will_be_submitted() works within tests
     if result_will_be_submitted:
         os.environ["RESULT_WILL_BE_SUBMITTED"] = "1"

tests/test_decorator.py

@@ -17,6 +17,7 @@
     extended_submission_test,
     run_tests,
     submission_test,
+    suppress_run_tests,
 )
 from ref_utils.error import RefUtilsError
 
@@ -394,3 +395,23 @@ def sub_test() -> bool:
         assert isinstance(results, list)
         assert len(results) == 1
         assert isinstance(results[0], TaskTestResult)
+
+    def test_suppress_run_tests_returns_empty_with_warning(self) -> None:
+        """Test that run_tests() emits a deprecation warning when suppressed."""
+
+        @submission_test()
+        def sub_test() -> bool:
+            return True
+
+        suppress_run_tests(True)
+        try:
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter("always")
+                results = run_tests()
+
+            assert results == []
+            assert len(w) == 1
+            assert issubclass(w[0].category, DeprecationWarning)
+            assert "deprecated" in str(w[0].message).lower()
+        finally:
+            suppress_run_tests(False)