[StdlibUnittest] Allow tests to require a new process

When testing runtime lookup changes, we want to make sure each test runs in isolation. Add a new modifier, `.requireOwnProcess()`, to enable tests to guarantee they’re run in isolation.

Author: Harlan Haskins

stdlib/private/StdlibUnittest/StdlibUnittest.swift

@@ -839,7 +839,7 @@ func _childProcess() {
     var stderr = _Stderr()
     print("\(_stdlibUnittestStreamPrefix);end", to: &stderr)
 
-    if !testSuite._testByName(testName).canReuseChildProcessAfterTest {
+    if testSuite._shouldShutDownChildProcess(forTestNamed: testName) {
       return
     }
   }
@@ -1107,7 +1107,7 @@ class _ParentProcess {
     // Check if the child has sent us "end" markers for the current test.
     if stdoutEnd && stderrEnd {
       var status: ProcessTerminationStatus?
-      if !testSuite._testByName(testName).canReuseChildProcessAfterTest {
+      if testSuite._shouldShutDownChildProcess(forTestNamed: testName) {
         status = _waitForChild()
         switch status! {
         case .exit(0):
@@ -1147,7 +1147,11 @@ class _ParentProcess {
       return (failed: false, ())
     }
 #endif
-    print("\(_stdlibUnittestStreamPrefix);shutdown", to: &_childStdin)
+    // If the child process expects an EOF, its stdin fd has already been closed and
+    // it will shut itself down automatically.
+    if !_childStdin.isClosed {
+      print("\(_stdlibUnittestStreamPrefix);shutdown", to: &_childStdin)
+    }
 
     var childCrashed = false
 
@@ -1209,6 +1213,9 @@ class _ParentProcess {
       if t.stdinText != nil {
         print("The test \(fullTestName) requires stdin input and can't be run in-process, marking as failed")
         _anyExpectFailed = true
+      } else if t.requiresOwnProcess {
+        print("The test \(fullTestName) requires running in a child process and can't be run in-process, marking as failed.")
+        _anyExpectFailed = true
       } else {
         _anyExpectFailed = false
         testSuite._runTest(name: t.name, parameter: testParameter)
@@ -1550,6 +1557,17 @@ public final class TestSuite {
     return _tests[_testNameToIndex[testName]!]
   }
 
+  /// Determines if we should shut down the current test process, i.e. if this
+  /// test or the next test requires executing in its own process.
+  func _shouldShutDownChildProcess(forTestNamed testName: String) -> Bool {
+    let index = _testNameToIndex[testName]!
+    if index == _tests.count - 1 { return false }
+    let currentTest = _tests[index]
+    let nextTest = _tests[index + 1]
+    if !currentTest.canReuseChildProcessAfterTest { return true }
+    return currentTest.requiresOwnProcess || nextTest.requiresOwnProcess
+  }
+
   internal enum _TestCode {
     case single(code: () -> Void)
     case parameterized(code: (Int) -> Void, count: Int)
@@ -1564,6 +1582,7 @@ public final class TestSuite {
     let stdinEndsWithEOF: Bool
     let crashOutputMatches: [String]
     let code: _TestCode
+    let requiresOwnProcess: Bool
 
     /// Whether the test harness should stop reusing the child process after
     /// running this test.
@@ -1601,6 +1620,7 @@ public final class TestSuite {
       var _stdinEndsWithEOF: Bool = false
       var _crashOutputMatches: [String] = []
       var _testLoc: SourceLoc?
+      var _requiresOwnProcess: Bool = false
     }
 
     init(testSuite: TestSuite, name: String, loc: SourceLoc) {
@@ -1630,6 +1650,11 @@ public final class TestSuite {
       return self
     }
 
+    public func requireOwnProcess() -> _TestBuilder {
+      _data._requiresOwnProcess = true
+      return self
+    }
+
     internal func _build(_ testCode: _TestCode) {
       _testSuite._tests.append(
         _Test(
@@ -1638,7 +1663,8 @@ public final class TestSuite {
           stdinText: _data._stdinText,
           stdinEndsWithEOF: _data._stdinEndsWithEOF,
           crashOutputMatches: _data._crashOutputMatches,
-          code: testCode))
+          code: testCode,
+          requiresOwnProcess: _data._requiresOwnProcess))
       _testSuite._testNameToIndex[_name] = _testSuite._tests.count - 1
     }
 

validation-test/StdlibUnittest/ChildProcessShutdown/RequireOwnProcess.swift

@@ -0,0 +1,41 @@
+// RUN: %target-run-simple-swift
+// REQUIRES: executable_test
+
+import StdlibUnittest
+#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS)
+  import Darwin
+#elseif os(Linux) || os(FreeBSD) || os(PS4) || os(Android) || os(Cygwin) || os(Haiku)
+  import Glibc
+#elseif os(Windows)
+  import MSVCRT
+#else
+#error("Unsupported platform")
+#endif
+
+//
+// Test that a test runs in its own child process if asked.
+//
+
+enum Globals {
+  static var modifiedByChildProcess = false
+}
+
+var TestSuiteRequireNewProcess = TestSuite("TestSuiteRequireNewProcess")
+
+TestSuiteRequireNewProcess.test("RequireOwnProcessBefore")
+  .code {
+  Globals.modifiedByChildProcess = true
+}
+
+TestSuiteRequireNewProcess.test("RequireOwnProcess")
+  .requireOwnProcess()
+  .code {
+  expectEqual(false, Globals.modifiedByChildProcess)
+  Globals.modifiedByChildProcess = true
+}
+
+TestSuiteRequireNewProcess.test("ShouldNotReusePreviousProcess") {
+  expectEqual(false, Globals.modifiedByChildProcess)
+}
+
+runAllTests()