Reactivate the integration tests for remote file access (GCS, S3) for non-conda environments

- 2 fake servers (fakes3 and fake-gcs-server) are used locally instead of a real S3 and GCS servers.
- The fake servers are provisioned with sample files before the tests run
- During the construction of the khiopsdev docker image we need to install the s3/gcs drivers required by khiops
- As these drivers are not released yet as conda packages we must disable the tests in conda environments (except when using a docker runner)

Author: Thierry RAMORASOAVINA

.github/workflows/dev-docker.yml

@@ -5,6 +5,8 @@ env:
   DEFAULT_IMAGE_INCREMENT: 0
   DEFAULT_SERVER_REVISION: main
   DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12 3.13
+  DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.10
+  DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.12
 on:
   pull_request:
     paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml]
@@ -34,6 +36,14 @@ on:
         type: string
         default: main
         description: Khiops Server Revision
+      khiops-gcs-driver-revision:
+        type: string
+        default: 0.0.10
+        description: Driver version for Google Cloud Storage remote files
+      khiops-s3-driver-revision:
+        type: string
+        default: 0.0.12
+        description: Driver version for AWS-S3 remote files
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -55,6 +65,8 @@ jobs:
           echo "KHIOPSDEV_OS_CODENAME=$(echo '${{ matrix.khiopsdev-os }}' | tr -d '0-9.')" >> "$GITHUB_ENV"
           echo "SERVER_REVISION=${{ inputs.server-revision || env.DEFAULT_SERVER_REVISION }}" >> "$GITHUB_ENV"
           echo "IMAGE_URL=ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.khiopsdev-os }}" >> "$GITHUB_ENV"
+          echo "KHIOPS_GCS_DRIVER_REVISION=${{ inputs.khiops-gcs-driver-revision || env.DEFAULT_KHIOPS_GCS_DRIVER_REVISION }}" >> "$GITHUB_ENV"
+          echo "KHIOPS_S3_DRIVER_REVISION=${{ inputs.khiops-s3-driver-revision || env.DEFAULT_KHIOPS_S3_DRIVER_REVISION }}" >> "$GITHUB_ENV"
       - name: Checkout khiops-python sources
         uses: actions/checkout@v4
       - name: Set up Docker Buildx
@@ -81,13 +93,18 @@ jobs:
       - name: Build image and push it to GitHub Container Registry
         uses: docker/build-push-action@v5
         with:
+          # Special hostname used by the integration tests for remote file access
+          # added using inputs because /etc/hosts is read-only for alternate builders (buildx via moby buildkit)
+          add-hosts: s3-bucket.localhost:127.0.0.1
           context: ./packaging/docker/khiopspydev/
           file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.KHIOPSDEV_OS_CODENAME }}
           build-args: |
             "KHIOPS_REVISION=${{ env.KHIOPS_REVISION }}"
             "KHIOPSDEV_OS=${{ matrix.khiopsdev-os }}"
             "SERVER_REVISION=${{ env.SERVER_REVISION }}"
             "PYTHON_VERSIONS=${{ inputs.python-versions || env.DEFAULT_PYTHON_VERSIONS }}"
+            "KHIOPS_GCS_DRIVER_REVISION=${{ env.KHIOPS_GCS_DRIVER_REVISION }}"
+            "KHIOPS_S3_DRIVER_REVISION=${{ env.KHIOPS_S3_DRIVER_REVISION }}"
           tags: ${{ env.DOCKER_IMAGE_TAGS }}
           # Push only on manual request
           push: ${{ inputs.push || false }}

.github/workflows/unit-tests.yml .github/workflows/tests.yml.github/workflows/unit-tests.yml renamed to .github/workflows/tests.yml

@@ -1,5 +1,5 @@
 ---
-name: Unit Tests
+name: Tests
 env:
   DEFAULT_SAMPLES_REVISION: 10.2.4
   DEFAULT_KHIOPS_DESKTOP_REVISION: 10.2.4
@@ -112,7 +112,32 @@ jobs:
       - name: Prepare Unit Tests Environment
         if: github.ref != 'dev' && github.ref != 'main' && ! inputs.run-long-tests
         run: echo "UNITTEST_ONLY_SHORT_TESTS=true" >> "$GITHUB_ENV"
-      - name: Run Unit Tests
+      - name: Prepare Integration Tests on remote files
+        env:
+          AWS_ENDPOINT_URL: http://localhost:4569
+        shell: bash
+        run: |
+          # Prepare AWS-S3 credentials and configuration          
+          mkdir -p ${GITHUB_WORKSPACE}/.aws/
+          cat << EOF > ${GITHUB_WORKSPACE}/.aws/credentials
+          [default]
+          aws_access_key_id=KEY
+          aws_secret_access_key=SECRET           
+          EOF
+          cat << EOF > ${GITHUB_WORKSPACE}/.aws/configuration
+          [default]
+          endpoint_url=${AWS_ENDPOINT_URL}
+          region=eu-north-1          
+          EOF
+          echo "Generated AWS credentials..."
+          cat  ${GITHUB_WORKSPACE}/.aws/credentials
+          echo "Generated AWS configuration..."
+          cat  ${GITHUB_WORKSPACE}/.aws/configuration
+
+          # Prepare GCS credentials
+          touch ${GITHUB_WORKSPACE}/google-credentials.json
+          /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
+      - name: Run Unit & Integration Tests
         env:
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
           KHIOPS_DOCKER_RUNNER_URL: https://localhost:11000
@@ -124,6 +149,21 @@ jobs:
           rmaps_base_oversubscribe: true
           # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
+          # for the tests with GCS
+          GCS_BUCKET_NAME: gcs-bucket
+          # we take advantage of the built-in `STORAGE_EMULATOR_HOST` env variable
+          # that every GCS client can read and lets us use a local fake file server
+          STORAGE_EMULATOR_HOST: http://localhost:4443
+          # even in GCS emulation mode, the credentials file must exist
+          GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/google-credentials.json
+          GCS_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
+          S3_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
+          # for the tests with S3
+          S3_BUCKET_NAME: s3-bucket
+          AWS_SHARED_CREDENTIALS_FILE: ${{ github.workspace }}/.aws/credentials
+          AWS_CONFIG_FILE: ${{ github.workspace }}/.aws/configuration
+          # common var for tests with GCS & S3
+          no_proxy: localhost
         run: |
           # This is needed so that the Git tag is parsed and the khiops-python
           # version is retrieved
@@ -138,10 +178,10 @@ jobs:
             $CONDA run --no-capture-output -n "$CONDA_ENV" coverage report -m
             $CONDA run --no-capture-output -n "$CONDA_ENV" coverage xml -o "reports/$CONDA_ENV/py-coverage.xml"
           done
-      - name: Display Unit Test Reports
+      - name: Display Test Reports
         uses: dorny/test-reporter@v1
         with:
-          name: Unit Tests ${{ matrix.python-version }}
+          name: Run Tests ${{ matrix.python-version }}
           path: >-
             reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml,
             reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml

packaging/conda/meta.yaml

@@ -30,8 +30,12 @@ requirements:
     - pandas >=0.25.3
     - scikit-learn >=0.22.2
   run_constrained:
-    - boto3 >=1.17.39
+    # do not necessary use the latest version
+    # to avoid undesired breaking changes
+    - boto3 >=1.17.39,<=1.35.69
     - google-cloud-storage >=1.37.0
+    # an open issue on boto3 (https://github.com/boto/boto3/issues/3585) forces a min version of pyopenssl
+    - pyopenssl>=24.0.0,<25.0.0
 
 outputs:
   - name: {{ metadata.get('name') }}

packaging/docker/khiopspydev/Dockerfile.ubuntu

@@ -10,14 +10,15 @@ ARG KHIOPS_REVISION
 RUN true \
   # Install git (for khiops-python version calculation) and pip \
   && apt-get -y update \
-  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget \
+  && apt-get -y --no-install-recommends install git python3-pip zip pandoc wget ruby-dev \
   # Get Linux distribution codename \
   && if [ -f /etc/os-release ]; then . /etc/os-release; fi \
   # Obtain the Khiops native package \
   && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \
   && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \
-  # Install the Khiops native package \
-  && dpkg -i --force-all KHIOPS_CORE.deb \
+  # Install the Khiops native package : make it always succeed. \
+  # If dpkg fails it is due to missing dependencies which will be installed by apt in the next line \
+  && (dpkg -i --force-all KHIOPS_CORE.deb || true) \
   && apt-get -f -y install \
   && rm -f KHIOPS_CORE.deb \
   # Set python to python3 \
@@ -53,10 +54,36 @@ RUN true \
   && true
 
 RUN mkdir -p /scripts
-COPY ./run_service.sh /scripts/run_service.sh
-RUN chmod +x /scripts/run_service.sh && \
+COPY ./run_service.sh ./run_fake_remote_file_servers.sh /scripts/
+RUN chmod +x /scripts/run_service.sh /scripts/run_fake_remote_file_servers.sh && \
   useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu
 
+# new gcs & s3 drivers (written in C++)
+ARG KHIOPS_GCS_DRIVER_REVISION
+ARG KHIOPS_S3_DRIVER_REVISION
+RUN true \
+    && wget -O khiops-gcs.deb https://github.com/KhiopsML/khiopsdriver-gcs/releases/download/${KHIOPS_GCS_DRIVER_REVISION}/khiops-gcs_${KHIOPS_GCS_DRIVER_REVISION}.deb \
+    && wget -O khiops-s3.deb https://github.com/KhiopsML/khiopsdriver-s3/releases/download/${KHIOPS_S3_DRIVER_REVISION}/khiops-s3_${KHIOPS_S3_DRIVER_REVISION}.deb \
+    && (dpkg -i --force-all khiops-gcs.deb khiops-s3.deb || true) \
+    && apt-get -f -y install \
+    && rm -f khiops-gcs.deb khiops-s3.deb \
+    && true
+
 FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server
+
 FROM khiopsdev AS base
 COPY --from=server /service /usr/bin/service
+
+# S3 fake file server (only in the ubuntu container)
+# Do not use the latest fakes3 version because starting from 1.3 a licence is required
+# if fakes3 is no longer compatible think about switching to an alternative and fully compatible server
+# (https://github.com/jamhall/s3rver:v3.7.1 is not yet for example)
+RUN gem install fakes3:1.2.1 sorted_set
+# Avoid resolving a fake s3-bucket.localhost hostname
+# Alternate builders (buildx via moby buildkit) mount /etc/hosts read-only, the following command will fail
+# echo "127.0.0.1 s3-bucket.localhost" >> /etc/hosts
+# You will have to add the `add-hosts` input instead (https://github.com/docker/build-push-action/#inputs)
+
+# Port on which fakes3 is listening
+EXPOSE 4569
+

packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+ROOT_FOLDER=${1:-.} # defaults to current folder
+
+# File server for S3 (runs in background)
+# WARNING :
+# -r : exposes pre-provisioned files (not currently used feature) : the direct child folder will be the bucket name
+#      these files were uploaded once because fake-s3 creates metadata
+echo "Launching fakes3 in background..."
+PORT_NUMBER=${AWS_ENDPOINT_URL##*:}
+nohup /usr/local/bin/fakes3 \
+  -r "${ROOT_FOLDER}"/tests/resources/remote-access \
+  -p "${PORT_NUMBER}" > /dev/null < /dev/null 2>&1 & # needs to redirect all the 3 fds to free the TTY

setup.py

@@ -49,7 +49,14 @@
         ],
         cmdclass=versioneer.get_cmdclass(),
         extras_require={
-            "s3": ["boto3>=1.17.39"],
+            "s3": [
+                # do not necessary use the latest version
+                # to avoid undesired breaking changes
+                "boto3>=1.17.39,<=1.35.69",
+                # an open issue on boto3 (https://github.com/boto/boto3/issues/3585)
+                # forces a minimal version of pyopenssl
+                "pyopenssl>=24.0.0,<25.0.0",
+            ],
             "gcs": ["google-cloud-storage>=1.37.0"],
         },
     )