Use the C++ drivers for remote file access (GCS, S3) instead of the old golang ones

- the GCS tests use a real Google account (via "Workload Identity Federation" authentication mode)
- the S3 ones still use an emulated S3 server

Author: Thierry RAMORASOAVINA

.github/workflows/dev-docker.yml

@@ -5,8 +5,8 @@ env:
   DEFAULT_IMAGE_INCREMENT: 0
   DEFAULT_SERVER_REVISION: main
   DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12 3.13
-  DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.10
-  DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.12
+  DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.11
+  DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.13
 on:
   pull_request:
     paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml]
@@ -38,11 +38,11 @@ on:
         description: Khiops Server Revision
       khiops-gcs-driver-revision:
         type: string
-        default: 0.0.10
+        default: 0.0.11
         description: Driver version for Google Cloud Storage remote files
       khiops-s3-driver-revision:
         type: string
-        default: 0.0.12
+        default: 0.0.13
         description: Driver version for AWS-S3 remote files
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/tests.yml

@@ -133,10 +133,19 @@ jobs:
           cat  ${GITHUB_WORKSPACE}/.aws/credentials
           echo "Generated AWS configuration..."
           cat  ${GITHUB_WORKSPACE}/.aws/configuration
-
-          # Prepare GCS credentials
-          touch ${GITHUB_WORKSPACE}/google-credentials.json
           /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
+      - name: Authenticate to GCP using "Workload Identity Federation"
+        # For integration tests on GCS we use a real Google account
+        # Retrieve the Google credentials through "Workload Identity Federation"
+        # see https://github.com/google-github-actions/auth?tab=readme-ov-file#workload-identity-federation-through-a-service-account
+        uses: google-github-actions/auth@v2
+        with:
+          service_account: khiops-gcs-driver-test-sa@ino-olr-dak-ideal-sbx.iam.gserviceaccount.com
+          workload_identity_provider: projects/322269704080/locations/global/workloadIdentityPools/github/providers/my-repo
+          # 'create_credentials_file' is true by default but let's make it explicit
+          # After authentication, the required GOOGLE_APPLICATION_CREDENTIALS environment variable is exported
+          # https://github.com/google-github-actions/auth?tab=readme-ov-file#inputs-miscellaneous
+          create_credentials_file: true
       - name: Run Unit & Integration Tests
         env:
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
@@ -150,19 +159,14 @@ jobs:
           # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
           # for the tests with GCS
-          GCS_BUCKET_NAME: gcs-bucket
-          # we take advantage of the built-in `STORAGE_EMULATOR_HOST` env variable
-          # that every GCS client can read and lets us use a local fake file server
-          STORAGE_EMULATOR_HOST: http://localhost:4443
-          # even in GCS emulation mode, the credentials file must exist
-          GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/google-credentials.json
+          GCS_BUCKET_NAME: data-test-khiops-driver-gcs/khiops_data
           GCS_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
-          S3_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
           # for the tests with S3
+          S3_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
           S3_BUCKET_NAME: s3-bucket
           AWS_SHARED_CREDENTIALS_FILE: ${{ github.workspace }}/.aws/credentials
           AWS_CONFIG_FILE: ${{ github.workspace }}/.aws/configuration
-          # common var for tests with GCS & S3
+          # Var for tests with S3
           no_proxy: localhost
         run: |
           # This is needed so that the Git tag is parsed and the khiops-python

packaging/docker/khiopspydev/Dockerfile.ubuntu

@@ -40,6 +40,8 @@ RUN true \
 # set up all the supported Python environments under conda (for the unit tests)
 # relying on a variable containing all the versions
 ARG PYTHON_VERSIONS
+ARG KHIOPS_GCS_DRIVER_REVISION
+ARG KHIOPS_S3_DRIVER_REVISION
 RUN true \
   && export CONDA="/root/miniforge3/bin/conda" \
   && /bin/bash -c 'for version in ${PYTHON_VERSIONS}; \
@@ -49,7 +51,12 @@ RUN true \
         do \
             $CONDA create -y -n $CONDA_ENV python=${version}; \
         done; \
+        # khiops core \
         $CONDA install -y -n py${version}_conda -c khiops-dev khiops-core=$(echo ${KHIOPS_REVISION} | tr -d "-") ; \
+        # remote files drivers installed in the conda environment \
+        $CONDA install -y -n py${version}_conda -c khiops \
+        khiops-driver-s3=${KHIOPS_S3_DRIVER_REVISION} \
+        khiops-driver-gcs=${KHIOPS_GCS_DRIVER_REVISION}; \
     done' \
   && true
 
@@ -58,12 +65,12 @@ COPY ./run_service.sh ./run_fake_remote_file_servers.sh /scripts/
 RUN chmod +x /scripts/run_service.sh /scripts/run_fake_remote_file_servers.sh && \
   useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu
 
-# new gcs & s3 drivers (written in C++)
-ARG KHIOPS_GCS_DRIVER_REVISION
-ARG KHIOPS_S3_DRIVER_REVISION
+# remote files drivers installed system-wide
 RUN true \
-    && wget -O khiops-gcs.deb https://github.com/KhiopsML/khiopsdriver-gcs/releases/download/${KHIOPS_GCS_DRIVER_REVISION}/khiops-gcs_${KHIOPS_GCS_DRIVER_REVISION}.deb \
-    && wget -O khiops-s3.deb https://github.com/KhiopsML/khiopsdriver-s3/releases/download/${KHIOPS_S3_DRIVER_REVISION}/khiops-s3_${KHIOPS_S3_DRIVER_REVISION}.deb \
+    # Get Linux distribution codename \
+    && if [ -f /etc/os-release ]; then . /etc/os-release; fi \
+    && wget -O khiops-gcs.deb https://github.com/KhiopsML/khiopsdriver-gcs/releases/download/${KHIOPS_GCS_DRIVER_REVISION}/khiops-driver-gcs_${KHIOPS_GCS_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \
+    && wget -O khiops-s3.deb https://github.com/KhiopsML/khiopsdriver-s3/releases/download/${KHIOPS_S3_DRIVER_REVISION}/khiops-driver-s3_${KHIOPS_S3_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \
     && (dpkg -i --force-all khiops-gcs.deb khiops-s3.deb || true) \
     && apt-get -f -y install \
     && rm -f khiops-gcs.deb khiops-s3.deb \

tests/test_remote_access.py

@@ -105,12 +105,7 @@ def remote_access_test_case(self):
             return ""
 
         def should_skip_in_a_conda_env(self):
-            """To be overriden by descendants
-
-            Temporarily skip the S3 and GCS tests in a conda environment
-            until the drivers for remote files access
-            are released as conda packages
-            """
+            """To be overriden by descendants"""
             return True
 
         def print_test_title(self):
@@ -145,11 +140,9 @@ def is_in_a_conda_env():
             ) == os.path.dirname(khiops_path)
 
         def setUp(self):
+
             self.skip_if_no_config()
             if self.is_in_a_conda_env() and self.should_skip_in_a_conda_env():
-                # TODO : Temporarily skip the test for Conda-installed khiops-core
-                #  until the drivers for remote files access
-                #  are released as conda packages
                 self.skipTest(
                     f"Remote test case {self.remote_access_test_case()} "
                     "in a conda environment is currently skipped"
@@ -322,6 +315,11 @@ def tearDownClass(cls):
         if s3_config_exists():
             kh.get_runner().__init__()
 
+    def should_skip_in_a_conda_env(self):
+        # The S3 driver is now released for conda too.
+        # No need to skip the tests any longer in a conda environment
+        return False
+
     def config_exists(self):
         return s3_config_exists()
 
@@ -360,6 +358,11 @@ def tearDownClass(cls):
         if gcs_config_exists():
             kh.get_runner().__init__()
 
+    def should_skip_in_a_conda_env(self):
+        # The GCS driver is now released for conda too.
+        # No need to skip the tests any longer in a conda environment
+        return False
+
     def config_exists(self):
         return gcs_config_exists()