Merge pull request #210 from KhiopsML/175-integrationtests-gcs-s3-remote-files

Reactivate the integration tests for remote file access (GCS, S3)

Author: tramora

.github/workflows/dev-docker.yml

@@ -5,6 +5,8 @@ env:
   DEFAULT_IMAGE_INCREMENT: 0
   DEFAULT_SERVER_REVISION: main
   DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12 3.13
+  DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.11
+  DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.13
 on:
   pull_request:
     paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml]
@@ -34,6 +36,14 @@ on:
         type: string
         default: main
         description: Khiops Server Revision
+      khiops-gcs-driver-revision:
+        type: string
+        default: 0.0.11
+        description: Driver version for Google Cloud Storage remote files
+      khiops-s3-driver-revision:
+        type: string
+        default: 0.0.13
+        description: Driver version for AWS-S3 remote files
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -55,6 +65,8 @@ jobs:
           echo "KHIOPSDEV_OS_CODENAME=$(echo '${{ matrix.khiopsdev-os }}' | tr -d '0-9.')" >> "$GITHUB_ENV"
           echo "SERVER_REVISION=${{ inputs.server-revision || env.DEFAULT_SERVER_REVISION }}" >> "$GITHUB_ENV"
           echo "IMAGE_URL=ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.khiopsdev-os }}" >> "$GITHUB_ENV"
+          echo "KHIOPS_GCS_DRIVER_REVISION=${{ inputs.khiops-gcs-driver-revision || env.DEFAULT_KHIOPS_GCS_DRIVER_REVISION }}" >> "$GITHUB_ENV"
+          echo "KHIOPS_S3_DRIVER_REVISION=${{ inputs.khiops-s3-driver-revision || env.DEFAULT_KHIOPS_S3_DRIVER_REVISION }}" >> "$GITHUB_ENV"
       - name: Checkout khiops-python sources
         uses: actions/checkout@v4
       - name: Set up Docker Buildx
@@ -81,13 +93,18 @@ jobs:
       - name: Build image and push it to GitHub Container Registry
         uses: docker/build-push-action@v5
         with:
+          # Special hostname used by the integration tests for remote file access
+          # added using inputs because /etc/hosts is read-only for alternate builders (buildx via moby buildkit)
+          add-hosts: s3-bucket.localhost:127.0.0.1
           context: ./packaging/docker/khiopspydev/
           file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.KHIOPSDEV_OS_CODENAME }}
           build-args: |
             "KHIOPS_REVISION=${{ env.KHIOPS_REVISION }}"
             "KHIOPSDEV_OS=${{ matrix.khiopsdev-os }}"
             "SERVER_REVISION=${{ env.SERVER_REVISION }}"
             "PYTHON_VERSIONS=${{ inputs.python-versions || env.DEFAULT_PYTHON_VERSIONS }}"
+            "KHIOPS_GCS_DRIVER_REVISION=${{ env.KHIOPS_GCS_DRIVER_REVISION }}"
+            "KHIOPS_S3_DRIVER_REVISION=${{ env.KHIOPS_S3_DRIVER_REVISION }}"
           tags: ${{ env.DOCKER_IMAGE_TAGS }}
           # Push only on manual request
           push: ${{ inputs.push || false }}

.github/workflows/unit-tests.yml .github/workflows/tests.yml.github/workflows/unit-tests.yml renamed to .github/workflows/tests.yml

@@ -1,5 +1,5 @@
 ---
-name: Unit Tests
+name: Tests
 env:
   DEFAULT_SAMPLES_REVISION: 10.2.4
   DEFAULT_KHIOPS_DESKTOP_REVISION: 10.2.4
@@ -15,11 +15,11 @@ on:
       khiops-desktop-revision:
         default: 10.2.4
         description: Khiops Windows Desktop Application Version
-      run-long-tests:
+      run-expensive-tests:
         type: boolean
         required: false
         default: false
-        description: Execute long tests
+        description: Execute expensive tests
   pull_request:
     paths:
       - khiops/**.py
@@ -110,9 +110,43 @@ jobs:
             rm -rf khiops.egg-info
           done
       - name: Prepare Unit Tests Environment
-        if: github.ref != 'dev' && github.ref != 'main' && ! inputs.run-long-tests
-        run: echo "UNITTEST_ONLY_SHORT_TESTS=true" >> "$GITHUB_ENV"
-      - name: Run Unit Tests
+        if: github.ref != 'dev' && github.ref != 'main' && ! inputs.run-expensive-tests
+        run: echo "SKIP_EXPENSIVE_TESTS=true" >> "$GITHUB_ENV"
+      - name: Prepare Integration Tests on remote files
+        env:
+          AWS_ENDPOINT_URL: http://localhost:4569
+        shell: bash
+        run: |
+          # Prepare AWS-S3 credentials and configuration          
+          mkdir -p ${GITHUB_WORKSPACE}/.aws/
+          cat << EOF > ${GITHUB_WORKSPACE}/.aws/credentials
+          [default]
+          aws_access_key_id=KEY
+          aws_secret_access_key=SECRET           
+          EOF
+          cat << EOF > ${GITHUB_WORKSPACE}/.aws/configuration
+          [default]
+          endpoint_url=${AWS_ENDPOINT_URL}
+          region=eu-north-1          
+          EOF
+          echo "Generated AWS credentials..."
+          cat  ${GITHUB_WORKSPACE}/.aws/credentials
+          echo "Generated AWS configuration..."
+          cat  ${GITHUB_WORKSPACE}/.aws/configuration
+          /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
+      - name: Authenticate to GCP using "Workload Identity Federation"
+        # For integration tests on GCS we use a real Google account
+        # Retrieve the Google credentials through "Workload Identity Federation"
+        # see https://github.com/google-github-actions/auth?tab=readme-ov-file#workload-identity-federation-through-a-service-account
+        uses: google-github-actions/auth@v2
+        with:
+          service_account: khiops-gcs-driver-test-sa@ino-olr-dak-ideal-sbx.iam.gserviceaccount.com
+          workload_identity_provider: projects/322269704080/locations/global/workloadIdentityPools/github/providers/my-repo
+          # 'create_credentials_file' is true by default but let's make it explicit
+          # After authentication, the required GOOGLE_APPLICATION_CREDENTIALS environment variable is exported
+          # https://github.com/google-github-actions/auth?tab=readme-ov-file#inputs-miscellaneous
+          create_credentials_file: true
+      - name: Run Unit & Integration Tests
         env:
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
           KHIOPS_DOCKER_RUNNER_URL: https://localhost:11000
@@ -124,6 +158,16 @@ jobs:
           rmaps_base_oversubscribe: true
           # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
+          # for the tests with GCS
+          GCS_BUCKET_NAME: data-test-khiops-driver-gcs/khiops_data
+          GCS_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
+          # for the tests with S3
+          S3_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
+          S3_BUCKET_NAME: s3-bucket
+          AWS_SHARED_CREDENTIALS_FILE: ${{ github.workspace }}/.aws/credentials
+          AWS_CONFIG_FILE: ${{ github.workspace }}/.aws/configuration
+          # Var for tests with S3
+          no_proxy: localhost
         run: |
           # This is needed so that the Git tag is parsed and the khiops-python
           # version is retrieved
@@ -138,10 +182,10 @@ jobs:
             $CONDA run --no-capture-output -n "$CONDA_ENV" coverage report -m
             $CONDA run --no-capture-output -n "$CONDA_ENV" coverage xml -o "reports/$CONDA_ENV/py-coverage.xml"
           done
-      - name: Display Unit Test Reports
+      - name: Display Test Reports
         uses: dorny/test-reporter@v1
         with:
-          name: Unit Tests ${{ matrix.python-version }}
+          name: Run Tests ${{ matrix.python-version }}
           path: >-
             reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml,
             reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml

khiops/core/internals/filesystems.py

@@ -23,6 +23,7 @@
 try:
     import boto3
     import boto3.session
+    from boto3.exceptions import S3UploadFailedError
     from botocore.exceptions import ClientError
 
     boto3_import_error = None
@@ -254,7 +255,7 @@ def copy_from_local(uri_or_path, local_path):
     Raises
     ------
     RuntimeError
-        If there was a problem when removing.
+        If there was a problem when copying.
     """
     create_resource(uri_or_path).copy_from_local(local_path)
 
@@ -272,7 +273,7 @@ def copy_to_local(uri_or_path, local_path):
     Raises
     ------
     RuntimeError
-        If there was a problem when removing.
+        If there was a problem when copying.
     """
     create_resource(uri_or_path).copy_to_local(local_path)
 
@@ -668,29 +669,22 @@ def remove(self):
             )
 
     def copy_from_local(self, local_path):
-        response = self.s3_client.Bucket(self.uri_info.netloc).upload_file(
-            local_path, self.uri_info.path[1:]
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        copy_ok = 200 <= status_code <= 299
-        if not copy_ok:
-            raise RuntimeError(
-                f"S3 copy_from_local failed {self.uri} with code {status_code}: "
-                + json.dumps(response)
+        try:
+            self.s3_client.Bucket(self.uri_info.netloc).upload_file(
+                local_path, self.uri_info.path[1:]
             )
+        # normalize the raised exception
+        except S3UploadFailedError as exc:
+            raise RuntimeError(f"S3 copy_from_local failed {self.uri}") from exc
 
     def copy_to_local(self, local_path):
-        response = self.s3_client.Bucket(self.uri_info.netloc).download_file(
-            local_path, self.uri_info.path[1:]
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        copy_ok = 200 <= status_code <= 299
-        if not copy_ok:
-            raise RuntimeError(
-                f"S3 download failed {self.uri} with code {status_code}: "
-                + json.dumps(response)
+        try:
+            self.s3_client.Bucket(self.uri_info.netloc).download_file(
+                self.uri_info.path[1:], local_path
             )
-        return copy_ok
+        # normalize the raised exception
+        except S3UploadFailedError as exc:
+            raise RuntimeError(f"S3 download failed {self.uri}") from exc
 
     def list_dir(self):
         # Add an extra slash to the path to treat it as a folder

khiops/core/internals/runner.py

@@ -594,17 +594,26 @@ def _report_exit_status(
 
         # Create the message reporting the errors and warnings
         error_msg = ""
-        errors, fatal_errors, warning_messages = self._collect_errors(log_file_path)
-        if warning_messages:
-            error_msg += "Warnings in log:\n" + "".join(warning_messages)
-        if errors:
-            if error_msg:
-                error_msg += "\n"
-            error_msg += "Errors in log:\n" + "".join(errors)
-        if fatal_errors:
-            if error_msg:
-                error_msg += "\n"
-            error_msg += "Fatal errors in log:\n" + "".join(fatal_errors)
+        # If the log file exists: Collect the errors and warnings messages
+        if fs.exists(log_file_path):
+            errors, fatal_errors, warning_messages = self._collect_errors(log_file_path)
+            if warning_messages:
+                error_msg += "Warnings in log:\n" + "".join(warning_messages)
+            if errors:
+                if error_msg:
+                    error_msg += "\n"
+                error_msg += "Errors in log:\n" + "".join(errors)
+            if fatal_errors:
+                if error_msg:
+                    error_msg += "\n"
+                error_msg += "Fatal errors in log:\n" + "".join(fatal_errors)
+        # Otherwise warn that the log file is missing
+        else:
+            warnings.warn(
+                f"Log file not found after {tool_name} execution."
+                f"Path: {log_file_path}"
+            )
+            errors = fatal_errors = []
 
         # Add stdout to the warning message if non empty
         if stdout:

packaging/conda/meta.yaml

@@ -30,8 +30,12 @@ requirements:
     - pandas >=0.25.3
     - scikit-learn >=0.22.2
   run_constrained:
-    - boto3 >=1.17.39
+    # do not necessary use the latest version
+    # to avoid undesired breaking changes
+    - boto3 >=1.17.39,<=1.35.69
     - google-cloud-storage >=1.37.0
+    # an open issue on boto3 (https://github.com/boto/boto3/issues/3585) forces a min version of pyopenssl
+    - pyopenssl>=24.0.0,<25.0.0
 
 outputs:
   - name: {{ metadata.get('name') }}

packaging/docker/khiopspydev/Dockerfile.ubuntu

@@ -10,14 +10,15 @@ ARG KHIOPS_REVISION
 RUN true \
   # Install git (for khiops-python version calculation) and pip \
   && apt-get -y update \
-  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget \
+  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget ruby-dev \
   # Get Linux distribution codename \
   && if [ -f /etc/os-release ]; then . /etc/os-release; fi \
   # Obtain the Khiops native package \
   && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \
   && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \
-  # Install the Khiops native package \
-  && dpkg -i --force-all KHIOPS_CORE.deb \
+  # Install the Khiops native package : make it always succeed. \
+  # If dpkg fails it is due to missing dependencies which will be installed by apt in the next line \
+  && (dpkg -i --force-all KHIOPS_CORE.deb || true) \
   && apt-get -f -y install \
   && rm -f KHIOPS_CORE.deb \
   # Set python to python3 \
@@ -39,6 +40,8 @@ RUN true \
 # set up all the supported Python environments under conda (for the unit tests)
 # relying on a variable containing all the versions
 ARG PYTHON_VERSIONS
+ARG KHIOPS_GCS_DRIVER_REVISION
+ARG KHIOPS_S3_DRIVER_REVISION
 RUN true \
   && export CONDA="/root/miniforge3/bin/conda" \
   && /bin/bash -c 'for version in ${PYTHON_VERSIONS}; \
@@ -48,15 +51,46 @@ RUN true \
         do \
             $CONDA create -y -n $CONDA_ENV python=${version}; \
         done; \
+        # khiops core \
         $CONDA install -y -n py${version}_conda -c khiops-dev khiops-core=$(echo ${KHIOPS_REVISION} | tr -d "-") ; \
+        # remote files drivers installed in the conda environment \
+        $CONDA install -y -n py${version}_conda -c khiops \
+        khiops-driver-s3=${KHIOPS_S3_DRIVER_REVISION} \
+        khiops-driver-gcs=${KHIOPS_GCS_DRIVER_REVISION}; \
     done' \
   && true
 
 RUN mkdir -p /scripts
-COPY ./run_service.sh /scripts/run_service.sh
-RUN chmod +x /scripts/run_service.sh && \
+COPY ./run_service.sh ./run_fake_remote_file_servers.sh /scripts/
+RUN chmod +x /scripts/run_service.sh /scripts/run_fake_remote_file_servers.sh && \
   useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu
 
+# remote files drivers installed system-wide
+RUN true \
+    # Get Linux distribution codename \
+    && if [ -f /etc/os-release ]; then . /etc/os-release; fi \
+    && wget -O khiops-gcs.deb https://github.com/KhiopsML/khiopsdriver-gcs/releases/download/${KHIOPS_GCS_DRIVER_REVISION}/khiops-driver-gcs_${KHIOPS_GCS_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \
+    && wget -O khiops-s3.deb https://github.com/KhiopsML/khiopsdriver-s3/releases/download/${KHIOPS_S3_DRIVER_REVISION}/khiops-driver-s3_${KHIOPS_S3_DRIVER_REVISION}-1-${VERSION_CODENAME}.amd64.deb \
+    && (dpkg -i --force-all khiops-gcs.deb khiops-s3.deb || true) \
+    && apt-get -f -y install \
+    && rm -f khiops-gcs.deb khiops-s3.deb \
+    && true
+
 FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server
+
 FROM khiopsdev AS base
 COPY --from=server /service /usr/bin/service
+
+# S3 fake file server (only in the ubuntu container)
+# Do not use the latest fakes3 version because starting from 1.3 a licence is required
+# if fakes3 is no longer compatible think about switching to an alternative and fully compatible server
+# (https://github.com/jamhall/s3rver:v3.7.1 is not yet for example)
+RUN gem install fakes3:1.2.1 sorted_set
+# Avoid resolving a fake s3-bucket.localhost hostname
+# Alternate builders (buildx via moby buildkit) mount /etc/hosts read-only, the following command will fail
+# echo "127.0.0.1 s3-bucket.localhost" >> /etc/hosts
+# You will have to add the `add-hosts` input instead (https://github.com/docker/build-push-action/#inputs)
+
+# Port on which fakes3 is listening
+EXPOSE 4569
+

packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+ROOT_FOLDER=${1:-.} # defaults to current folder
+
+# File server for S3 (runs in background)
+# WARNING :
+# -r : exposes pre-provisioned files (not currently used feature) : the direct child folder will be the bucket name
+#      these files were uploaded once because fake-s3 creates metadata
+echo "Launching fakes3 in background..."
+PORT_NUMBER=${AWS_ENDPOINT_URL##*:}
+nohup /usr/local/bin/fakes3 \
+  -r "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -p "${PORT_NUMBER}" > /dev/null < /dev/null 2>&1 & # needs to redirect all the 3 fds to free the TTY

setup.py

@@ -49,7 +49,14 @@
         ],
         cmdclass=versioneer.get_cmdclass(),
         extras_require={
-            "s3": ["boto3>=1.17.39"],
+            "s3": [
+                # do not necessary use the latest version
+                # to avoid undesired breaking changes
+                "boto3>=1.17.39,<=1.35.69",
+                # an open issue on boto3 (https://github.com/boto/boto3/issues/3585)
+                # forces a minimal version of pyopenssl
+                "pyopenssl>=24.0.0,<25.0.0",
+            ],
             "gcs": ["google-cloud-storage>=1.37.0"],
         },
     )

tests/test_helper.py

@@ -272,10 +272,10 @@ def get_with_subkey(dictionary, subkey):
         return values
 
     @staticmethod
-    def skip_long_test(test_case):
-        if "UNITTEST_ONLY_SHORT_TESTS" in os.environ:
-            if os.environ["UNITTEST_ONLY_SHORT_TESTS"].lower() == "true":
-                test_case.skipTest("Skipping long test")
+    def skip_expensive_test(test_case):
+        if "SKIP_EXPENSIVE_TESTS" in os.environ:
+            if os.environ["SKIP_EXPENSIVE_TESTS"].lower() == "true":
+                test_case.skipTest("Skipping expensive test")
 
     @staticmethod
     def create_parameter_trace():