PHPLIB-891: Run CSFLE tests with crypt_shared (#943)

* Ensure TESTS var is passed through to run-tests.sh

This likely dates back to d1b18dd

Author: jmikola

.evergreen/config.yml

@@ -246,8 +246,9 @@ functions:
           export KMS_ENDPOINT_REQUIRE_CLIENT_CERT="${client_side_encryption_kms_endpoint_require_client_cert}"
           export KMS_TLS_CA_FILE="${client_side_encryption_kms_tls_ca_file}"
           export KMS_TLS_CERTIFICATE_KEY_FILE="${client_side_encryption_kms_tls_certificate_key_file}"
+          export CRYPT_SHARED_LIB_PATH="${client_side_encryption_crypt_shared_lib_path}"
           export PATH="${PHP_PATH}/bin:$PATH"
-          API_VERSION=${API_VERSION} PHP_VERSION=${PHP_VERSION} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
+          API_VERSION=${API_VERSION} PHP_VERSION=${PHP_VERSION} TESTS=${TESTS} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run atlas data lake test":
      - command: shell.exec
@@ -377,6 +378,18 @@ functions:
           - key: client_side_encryption_kmip_endpoint
             value: localhost:5698
 
+  "fetch crypt_shared":
+    - command: shell.exec
+      params:
+        script: |
+          # TODO: Specify same version provisioned by download-mongodb.sh (see: DRIVERS-2355)
+          python3 ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component crypt_shared --version latest --only "**/mongo_crypt_v1.so" --out ${DRIVERS_TOOLS}/.evergreen/csfle --strip-path-components 1
+    - command: expansions.update
+      params:
+        updates:
+          - key: client_side_encryption_crypt_shared_lib_path
+            value: ${DRIVERS_TOOLS}/.evergreen/csfle/mongo_crypt_v1.so
+
 pre:
   - func: "fetch source"
   - func: "prepare resources"
@@ -507,6 +520,18 @@ tasks:
             MONGODB_URI: "${SINGLE_MONGOS_LB_URI}"
             SSL: "yes"
         # Note: "stop load balancer" will be called from "post"
+
+    - name: "test-crypt_shared"
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            TOPOLOGY: "replica_set"
+        - func: "start kms servers"
+        - func: "fetch crypt_shared"
+        - func: "run tests"
+          vars:
+            TESTS: "csfle"
+
 # }}}
 
 
@@ -766,3 +791,10 @@ buildvariants:
   display_name: "Load balanced - ${mongodb-versions}"
   tasks:
     - name: "test-loadBalanced"
+
+# CSFLE crypt_shared is available from MongoDB 6.0+
+- matrix_name: "test-csfle-crypt_shared"
+  matrix_spec: { "os": "debian11", "mongodb-versions": "6.0", "php-edge-versions": "latest-stable", "driver-versions": "latest-stable" }
+  display_name: "CSFLE crypt_shared - ${mongodb-versions}"
+  tasks:
+    - name: "test-crypt_shared"

.evergreen/run-tests.sh

@@ -67,6 +67,10 @@ case "$TESTS" in
       php vendor/bin/simple-phpunit --configuration phpunit.evergreen.xml --testsuite "Atlas Data Lake Test Suite" $PHPUNIT_OPTS
       ;;
 
+   csfle)
+      php vendor/bin/simple-phpunit --configuration phpunit.evergreen.xml --group csfle $PHPUNIT_OPTS
+      ;;
+
    versioned-api)
       php vendor/bin/simple-phpunit --configuration phpunit.evergreen.xml --group versioned-api $PHPUNIT_OPTS
       ;;

tests/SpecTests/ClientSideEncryptionSpecTest.php

@@ -31,6 +31,7 @@
 use function glob;
 use function in_array;
 use function is_executable;
+use function is_readable;
 use function iterator_to_array;
 use function json_decode;
 use function sprintf;
@@ -46,6 +47,7 @@
  * Client-side encryption spec tests.
  *
  * @see https://github.com/mongodb/specifications/tree/master/source/client-side-encryption
+ * @group csfle
  */
 class ClientSideEncryptionSpecTest extends FunctionalTestCase
 {
@@ -65,7 +67,10 @@ public function setUp(): void
         parent::setUp();
 
         $this->skipIfClientSideEncryptionIsNotSupported();
-        $this->skipIfLocalMongocryptdIsUnavailable();
+
+        if (! static::isCryptSharedLibAvailable() && ! static::isMongocryptdAvailable()) {
+            $this->markTestSkipped('Neither crypt_shared nor mongocryptd are available');
+        }
     }
 
     /**
@@ -79,6 +84,15 @@ public static function assertCommandMatches(stdClass $expected, stdClass $actual
         static::assertDocumentsMatch($expected, $actual);
     }
 
+    public static function createTestClient(?string $uri = null, array $options = [], array $driverOptions = []): Client
+    {
+        if (isset($driverOptions['autoEncryption']) && getenv('CRYPT_SHARED_LIB_PATH')) {
+            $driverOptions['autoEncryption']['extraOptions']['cryptSharedLibPath'] = getenv('CRYPT_SHARED_LIB_PATH');
+        }
+
+        return parent::createTestClient($uri, $options, $driverOptions);
+    }
+
     /**
      * Execute an individual test case from the specification.
      *
@@ -825,6 +839,14 @@ static function (self $test, ClientEncryption $clientEncryption, ClientEncryptio
      */
     public function testBypassSpawningMongocryptdViaBypassSpawn(): void
     {
+        /* If crypt_shared is available it will likely already have been loaded
+         * by a previous test so there is no way to prevent it from being used.
+         * Since CSFLE prefers crypt_shared to mongocryptd there is reason to
+         * run any of the "bypass spawning" tests (see also: MONGOCRYPT-421). */
+        if (static::isCryptSharedLibAvailable()) {
+            $this->markTestSkipped('Bypass spawning of mongocryptd cannot be tested when crypt_shared is available');
+        }
+
         $autoEncryptionOpts = [
             'keyVaultNamespace' => 'keyvault.datakeys',
             'kmsProviders' => [
@@ -840,6 +862,7 @@ public function testBypassSpawningMongocryptdViaBypassSpawn(): void
             ],
         ];
 
+        // Disable adding cryptSharedLibPath, as it may interfere with this test
         $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         try {
@@ -860,6 +883,10 @@ public function testBypassSpawningMongocryptdViaBypassSpawn(): void
      */
     public function testBypassSpawningMongocryptdViaBypassAutoEncryption(): void
     {
+        if (static::isCryptSharedLibAvailable()) {
+            $this->markTestSkipped('Bypass spawning of mongocryptd cannot be tested when crypt_shared is available');
+        }
+
         $autoEncryptionOpts = [
             'keyVaultNamespace' => 'keyvault.datakeys',
             'kmsProviders' => [
@@ -871,6 +898,7 @@ public function testBypassSpawningMongocryptdViaBypassAutoEncryption(): void
             ],
         ];
 
+        // Disable adding cryptSharedLibPath, as it may interfere with this test
         $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         $clientEncrypted->selectCollection('db', 'coll')->insertOne(['unencrypted' => 'test']);
@@ -888,6 +916,10 @@ public function testBypassSpawningMongocryptdViaBypassAutoEncryption(): void
      */
     public function testBypassSpawningMongocryptdViaBypassQueryAnalysis(): void
     {
+        if (static::isCryptSharedLibAvailable()) {
+            $this->markTestSkipped('Bypass spawning of mongocryptd cannot be tested when crypt_shared is available');
+        }
+
         $autoEncryptionOpts = [
             'keyVaultNamespace' => 'keyvault.datakeys',
             'kmsProviders' => [
@@ -899,6 +931,7 @@ public function testBypassSpawningMongocryptdViaBypassQueryAnalysis(): void
             ],
         ];
 
+        // Disable adding cryptSharedLibPath, as it may interfere with this test
         $clientEncrypted = static::createTestClient(null, [], ['autoEncryption' => $autoEncryptionOpts]);
 
         $clientEncrypted->selectCollection('db', 'coll')->insertOne(['unencrypted' => 'test']);
@@ -1525,16 +1558,27 @@ private function prepareEncryptedFieldsMap(stdClass $encryptedFieldsMap): stdCla
         return $encryptedFieldsMap;
     }
 
-    private function skipIfLocalMongocryptdIsUnavailable(): void
+    private static function isCryptSharedLibAvailable(): bool
+    {
+        $cryptSharedLibPath = getenv('CRYPT_SHARED_LIB_PATH');
+
+        if ($cryptSharedLibPath === false) {
+            return false;
+        }
+
+        return is_readable($cryptSharedLibPath);
+    }
+
+    private static function isMongocryptdAvailable(): bool
     {
         $paths = explode(PATH_SEPARATOR, getenv("PATH"));
 
         foreach ($paths as $path) {
             if (is_executable($path . DIRECTORY_SEPARATOR . 'mongocryptd')) {
-                return;
+                return true;
             }
         }
 
-        $this->markTestSkipped('Mongocryptd is not available on the localhost');
+        return false;
     }
 }

tests/SpecTests/Context.php

@@ -107,6 +107,11 @@ public static function fromClientSideEncryption(stdClass $test, $databaseName, $
 
                 $autoEncryptionOptions['tlsOptions']->kmip = self::getKmsTlsOptions();
             }
+
+            // Intentionally ignore empty values for CRYPT_SHARED_LIB_PATH
+            if (getenv('CRYPT_SHARED_LIB_PATH')) {
+                $autoEncryptionOptions['extraOptions']['cryptSharedLibPath'] = getenv('CRYPT_SHARED_LIB_PATH');
+            }
         }
 
         if (isset($test->outcome->collection->name)) {