Merge pull request #353 from KhiopsML/dev-v10

Release 10.3.0.0

Author: popescu-v

.github/ISSUE_TEMPLATE.md …ISSUE_TEMPLATE/generic-issue-template.md.github/ISSUE_TEMPLATE.md renamed to .github/ISSUE_TEMPLATE/generic-issue-template.md .github/ISSUE_TEMPLATE.md renamed to .github/ISSUE_TEMPLATE/generic-issue-template.md

@@ -3,13 +3,13 @@ name: Conda Package
 env:
   # Note: The default Khiops version must never be an alpha release as they are
   #       ephemeral. To test alpha versions run the workflow manually.
-  DEFAULT_KHIOPS_CORE_VERSION: 10.2.4
+  DEFAULT_KHIOPS_CORE_VERSION: 10.3.0
   DEFAULT_SAMPLES_VERSION: 10.2.4
 on:
   workflow_dispatch:
     inputs:
       khiops-core-version:
-        default: 10.2.4
+        default: 10.3.0
         description: khiops-core version for testing
       khiops-samples-version:
         default: 10.2.4
@@ -51,7 +51,7 @@ jobs:
       - name: Install Miniforge
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniforge-version: latest
+          miniforge-version: 24.11.0-0
           python-version: '3.12'
           conda-remove-defaults: true
       - name: Install Dependency Requirements for Building Conda Packages
@@ -90,7 +90,7 @@ jobs:
       - name: Install Miniforge
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniforge-version: latest  # needed for macOS 13
+          miniforge-version: 24.11.0-0
           python-version: ${{ matrix.python-version }}
           conda-remove-defaults: true
       - name: Download Conda Package Artifact
@@ -185,7 +185,7 @@ jobs:
       - name: Install Miniforge
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniforge-version: latest
+          miniforge-version: 24.11.0-0
           python-version: '3.12'
           conda-remove-defaults: true
       - name: Install Requirement Packages
@@ -201,9 +201,9 @@ jobs:
           if [[ "$ANACONDA_CHANNEL" == "khiops" ]]
           then
             anaconda --token "${{ secrets.KHIOPS_ANACONDA_CHANNEL_TOKEN }}" upload \
-              --user "$ANACONDA_CHANNEL" ./khiops-conda/noarch/*.tar.bz2
+              --user "$ANACONDA_CHANNEL" ./khiops-conda/noarch/*.conda
           # For the dev channel: upload with forcing
           else
             anaconda --token "${{ secrets.KHIOPS_DEV_ANACONDA_CHANNEL_TOKEN }}" upload \
-              --user "$ANACONDA_CHANNEL" --force ./khiops-conda/noarch/*.tar.bz2
+              --user "$ANACONDA_CHANNEL" --force ./khiops-conda/noarch/*.conda
           fi

.github/workflows/conda.yml

@@ -1,18 +1,20 @@
 ---
 name: Dev Docker
 env:
-  DEFAULT_KHIOPS_REVISION: 10.2.4
+  DEFAULT_KHIOPS_REVISION: 10.3.0
   DEFAULT_IMAGE_INCREMENT: 0
   DEFAULT_SERVER_REVISION: main
   DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12 3.13
+  DEFAULT_KHIOPS_GCS_DRIVER_REVISION: 0.0.11
+  DEFAULT_KHIOPS_S3_DRIVER_REVISION: 0.0.13
 on:
   pull_request:
     paths: [packaging/docker/khiopspydev/Dockerfile.*, .github/workflows/dev-docker.yml]
   workflow_dispatch:
     inputs:
       khiops-revision:
         type: string
-        default: 10.2.4
+        default: 10.3.0
         description: Khiops Revision
       image-increment:
         type: number
@@ -34,6 +36,14 @@ on:
         type: string
         default: main
         description: Khiops Server Revision
+      khiops-gcs-driver-revision:
+        type: string
+        default: 0.0.11
+        description: Driver version for Google Cloud Storage remote files
+      khiops-s3-driver-revision:
+        type: string
+        default: 0.0.13
+        description: Driver version for AWS-S3 remote files
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -55,6 +65,8 @@ jobs:
           echo "KHIOPSDEV_OS_CODENAME=$(echo '${{ matrix.khiopsdev-os }}' | tr -d '0-9.')" >> "$GITHUB_ENV"
           echo "SERVER_REVISION=${{ inputs.server-revision || env.DEFAULT_SERVER_REVISION }}" >> "$GITHUB_ENV"
           echo "IMAGE_URL=ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.khiopsdev-os }}" >> "$GITHUB_ENV"
+          echo "KHIOPS_GCS_DRIVER_REVISION=${{ inputs.khiops-gcs-driver-revision || env.DEFAULT_KHIOPS_GCS_DRIVER_REVISION }}" >> "$GITHUB_ENV"
+          echo "KHIOPS_S3_DRIVER_REVISION=${{ inputs.khiops-s3-driver-revision || env.DEFAULT_KHIOPS_S3_DRIVER_REVISION }}" >> "$GITHUB_ENV"
       - name: Checkout khiops-python sources
         uses: actions/checkout@v4
       - name: Set up Docker Buildx
@@ -69,9 +81,9 @@ jobs:
       - name: Set up the Docker Image Tags
         run: |
           set -x
-          # Set latest only if requested and on "dev" or "main" branch
+          # Set latest only if requested and on "dev", "dev-v10" or "main" branch
           if [[ "${{ inputs.set-latest }}" == "true" &&
-                ("$GITHUB_REF_NAME" == "dev" || "$GITHUB_REF_NAME" == "main") ]]
+                ("$GITHUB_REF_NAME" == "dev" || "$GITHUB_REF_NAME" == "dev-v10" || "$GITHUB_REF_NAME" == "main") ]]
           then
             DOCKER_IMAGE_TAGS="$IMAGE_URL:latest,$IMAGE_URL:$KHIOPS_REVISION.$IMAGE_INCREMENT"
           else
@@ -81,13 +93,18 @@ jobs:
       - name: Build image and push it to GitHub Container Registry
         uses: docker/build-push-action@v5
         with:
+          # Special hostname used by the integration tests for remote file access
+          # added using inputs because /etc/hosts is read-only for alternate builders (buildx via moby buildkit)
+          add-hosts: s3-bucket.localhost:127.0.0.1
           context: ./packaging/docker/khiopspydev/
           file: ./packaging/docker/khiopspydev/Dockerfile.${{ env.KHIOPSDEV_OS_CODENAME }}
           build-args: |
             "KHIOPS_REVISION=${{ env.KHIOPS_REVISION }}"
             "KHIOPSDEV_OS=${{ matrix.khiopsdev-os }}"
             "SERVER_REVISION=${{ env.SERVER_REVISION }}"
             "PYTHON_VERSIONS=${{ inputs.python-versions || env.DEFAULT_PYTHON_VERSIONS }}"
+            "KHIOPS_GCS_DRIVER_REVISION=${{ env.KHIOPS_GCS_DRIVER_REVISION }}"
+            "KHIOPS_S3_DRIVER_REVISION=${{ env.KHIOPS_S3_DRIVER_REVISION }}"
           tags: ${{ env.DOCKER_IMAGE_TAGS }}
           # Push only on manual request
           push: ${{ inputs.push || false }}

.github/workflows/dev-docker.yml

@@ -121,7 +121,7 @@ jobs:
         with:
           name: pip-package
       - name: Upload Pip package to the release
-        uses: ncipollo/release-action@v1.14.0
+        uses: ncipollo/release-action@v1.15.0
         with:
           allowUpdates: true
           artifacts: khiops*.tar.gz

.github/workflows/pip.yml

@@ -1,8 +1,8 @@
 ---
-name: Unit Tests
+name: Tests
 env:
   DEFAULT_SAMPLES_REVISION: 10.2.4
-  DEFAULT_KHIOPS_DESKTOP_REVISION: 10.2.4
+  DEFAULT_KHIOPS_DESKTOP_REVISION: 10.3.0
 on:
   workflow_dispatch:
     inputs:
@@ -13,21 +13,21 @@ on:
         default: latest
         description: Development Docker Image Tag
       khiops-desktop-revision:
-        default: 10.2.4
+        default: 10.3.0
         description: Khiops Windows Desktop Application Version
-      run-long-tests:
+      run-expensive-tests:
         type: boolean
         required: false
         default: false
-        description: Execute long tests
+        description: Execute expensive tests
   pull_request:
     paths:
       - khiops/**.py
       - pykhiops/**.py  # TODO: Remove for Khiops 11
       - tests/**.py
       - tests/resources/**
       - '!tests/resources/**.md'
-      - .github/workflows/unit-tests.yml
+      - .github/workflows/tests.yml
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -109,10 +109,44 @@ jobs:
             $CONDA install -y -n "$CONDA_ENV" `grep -v "^\[" khiops.egg-info/requires.txt`
             rm -rf khiops.egg-info
           done
-      - name: Prepare Unit Tests Environment
-        if: github.ref != 'dev' && github.ref != 'main' && ! inputs.run-long-tests
-        run: echo "UNITTEST_ONLY_SHORT_TESTS=true" >> "$GITHUB_ENV"
-      - name: Run Unit Tests
+      - name: Prepare Tests Environment
+        if: github.ref != 'dev' && github.rev != 'dev-v10' && github.ref != 'main' && ! inputs.run-expensive-tests
+        run: echo "SKIP_EXPENSIVE_TESTS=true" >> "$GITHUB_ENV"
+      - name: Prepare Integration Tests on remote files
+        env:
+          AWS_ENDPOINT_URL: http://localhost:4569
+        shell: bash
+        run: |
+          # Prepare AWS-S3 credentials and configuration          
+          mkdir -p ${GITHUB_WORKSPACE}/.aws/
+          cat << EOF > ${GITHUB_WORKSPACE}/.aws/credentials
+          [default]
+          aws_access_key_id=KEY
+          aws_secret_access_key=SECRET           
+          EOF
+          cat << EOF > ${GITHUB_WORKSPACE}/.aws/configuration
+          [default]
+          endpoint_url=${AWS_ENDPOINT_URL}
+          region=eu-north-1          
+          EOF
+          echo "Generated AWS credentials..."
+          cat  ${GITHUB_WORKSPACE}/.aws/credentials
+          echo "Generated AWS configuration..."
+          cat  ${GITHUB_WORKSPACE}/.aws/configuration
+          /scripts/run_fake_remote_file_servers.sh .  # launch the servers in the background
+      - name: Authenticate to GCP using "Workload Identity Federation"
+        # For integration tests on GCS we use a real Google account
+        # Retrieve the Google credentials through "Workload Identity Federation"
+        # see https://github.com/google-github-actions/auth?tab=readme-ov-file#workload-identity-federation-through-a-service-account
+        uses: google-github-actions/auth@v2
+        with:
+          service_account: khiops-gcs-driver-test-sa@ino-olr-dak-ideal-sbx.iam.gserviceaccount.com
+          workload_identity_provider: projects/322269704080/locations/global/workloadIdentityPools/github/providers/my-repo
+          # 'create_credentials_file' is true by default but let's make it explicit
+          # After authentication, the required GOOGLE_APPLICATION_CREDENTIALS environment variable is exported
+          # https://github.com/google-github-actions/auth?tab=readme-ov-file#inputs-miscellaneous
+          create_credentials_file: true
+      - name: Run Unit & Integration Tests
         env:
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
           KHIOPS_DOCKER_RUNNER_URL: https://localhost:11000
@@ -124,6 +158,16 @@ jobs:
           rmaps_base_oversubscribe: true
           # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
+          # for the tests with GCS
+          GCS_BUCKET_NAME: data-test-khiops-driver-gcs/khiops_data
+          GCS_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
+          # for the tests with S3
+          S3_DRIVER_LOGLEVEL: info  # set to debug for diagnosis
+          S3_BUCKET_NAME: s3-bucket
+          AWS_SHARED_CREDENTIALS_FILE: ${{ github.workspace }}/.aws/credentials
+          AWS_CONFIG_FILE: ${{ github.workspace }}/.aws/configuration
+          # Var for tests with S3
+          no_proxy: localhost
         run: |
           # This is needed so that the Git tag is parsed and the khiops-python
           # version is retrieved
@@ -138,10 +182,10 @@ jobs:
             $CONDA run --no-capture-output -n "$CONDA_ENV" coverage report -m
             $CONDA run --no-capture-output -n "$CONDA_ENV" coverage xml -o "reports/$CONDA_ENV/py-coverage.xml"
           done
-      - name: Display Unit Test Reports
+      - name: Display Test Reports
         uses: dorny/test-reporter@v1
         with:
-          name: Unit Tests ${{ matrix.python-version }}
+          name: Run Tests ${{ matrix.python-version }}
           path: >-
             reports/py${{ matrix.python-version }}/TEST-tests.*.*.xml,
             reports/py${{ matrix.python-version }}_conda/TEST-tests.*.*.xml
@@ -214,7 +258,7 @@ jobs:
           git config --global --add safe.directory $(Resolve-Path '.' | % {$_.toString()})
           python setup.py egg_info
 
-          # Install the Pyrhon requirements
+          # Install the Python requirements
           Get-Content .\khiops.egg-info\requires.txt `
             | Select-String -Pattern '^\[' -NotMatch `
             | Select-String -Pattern '^$' -NotMatch `

.github/workflows/unit-tests.yml .github/workflows/tests.yml.github/workflows/unit-tests.yml renamed to .github/workflows/tests.yml

@@ -6,6 +6,24 @@
   - Example: 10.2.1.4 is the 5th version that supports khiops 10.2.1.
 - Internals: Changes in *Internals* sections are unlikely to be of interest for data scientists.
 
+## 10.3.0.0 - 2025-02-10
+
+### Fixed
+- (`core`) Dictionary file `.json` extension check in the `khiops.dictionary.read_dictionary_file`
+function.
+
+### Changed
+- (`sklearn`) The `train_test_split_dataset` helper has been moved from `khiops.utils` to
+`khiops.sklearn`.
+- (`sklearn`) The `transform_pairs` parameter of the `KhiopsEncoder` sklearn estimator has been
+renamed to `transform_type_pairs`.
+
+### Removed
+- (`sklearn`) The `is_fitted_` estimator attribute. The Scikit-learn `check_is_fitted` function
+can be used to test the fitted state of the estimators.
+- (`sklearn`) The `n_pairs` parameter of the `KhiopsRegressor` sklearn estimator. It was never
+supported.
+
 ## 10.2.4.0 - 2024-12-19
 
 ### Added
@@ -87,7 +105,7 @@
 
 ### Fixed
 - `core`
-  - Metric name search in estimator analyis report.
+  - Metric name search in estimator analysis report.
 
 ## 10.2.1.0 - 2024-03-26
 
@@ -219,7 +237,7 @@ Note: This release marks the open sourcing of Khiops:
   - Estimators now accept dataframes with numerical column indexes.
   - `KhiopsClassifier` now accepts integer target vectors.
   - `classes_` estimator attribute for `KhiopsClassifier` (available once fitted).
-  - `feature_names_out_` estimator attirbute for `KhiopsEncoder` (available once fitted).
+  - `feature_names_out_` estimator attribute for `KhiopsEncoder` (available once fitted).
   - `export_report_file` and `export_dictionary_file` to export Khiops report and dictionary files
     once the estimators are fitted.
   - `internal_sort` parameter for estimators that may be used to not sort the tables on the

CHANGELOG.md

@@ -15,7 +15,7 @@ Khiops Python library.
 
 # You'll also need a system-wide installation of pandoc (https://pandoc.org)
 
-# Execute this if there were non commited updates to samples.py or samples_sklearn.py:
+# Execute this if there were non committed updates to samples.py or samples_sklearn.py:
 # ./convert-samples-hook
 
 # To clean the html documentation
@@ -29,7 +29,7 @@ Khiops Python library.
 ./create-doc -d -t
 
 # To only execute Sphinx on updated reST resources
-# make html
+# sphinx-build -M html . _build/
 ```
 
 ## Sphinx

doc/Makefile

@@ -75,7 +75,7 @@ parse_commandline "$@"
 rm -v core/generated/*.rst
 rm -v sklearn/generated/*.rst
 rm -v internal/generated/*.rst
-make clean
+sphinx-build -M clean . _build
 
 if [[ $_arg_clean_tutorial == "on" ]]
 then

doc/README.md

@@ -1,3 +1,9 @@
+######################################################################################
+# Copyright (c) 2023-2025 Orange. All rights reserved.                               #
+# This software is distributed under the BSD 3-Clause-clear License, the text of     #
+# which is available at https://spdx.org/licenses/BSD-3-Clause-Clear.html or         #
+# see the "LICENSE.md" file for more details.                                        #
+######################################################################################
 """Khiops Python Sphinx configuration file"""
 
 import os
@@ -92,12 +98,11 @@
         "color-admonition-title-background--note": "#CC6100",
         "font-stack": "Helvetica Neue, Helvetica, sans-serif",
     },
-    "source_repository": "https://github.com/khiopsml/khiops/",
     # Sets the Github Icon (the SVG is embedded, copied from furo's repo)
     "footer_icons": [
         {
             "name": "GitHub",
-            "url": "https://github.com/khiopsml/khiops",
+            "url": "https://github.com/khiopsml/khiops-python",
             "html": """
                 <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
                     <path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path>