Merge pull request #507 from KhiopsML/506-fix-home-for-openmpi5

Ensure HOME is always set when running khiops core with OpenMPI 5+

Author: tramora

.github/workflows/dev-docker.yml

@@ -1,7 +1,7 @@
 ---
 name: Dev Docker
 env:
-  DEFAULT_KHIOPS_REVISION: 11.0.0-b.0
+  DEFAULT_KHIOPS_REVISION: 11.0.0-rc.1
   DEFAULT_IMAGE_INCREMENT: 0
   DEFAULT_SERVER_REVISION: main
   DEFAULT_PYTHON_VERSIONS: 3.8 3.9 3.10 3.11 3.12 3.13 3.14
@@ -14,7 +14,7 @@ on:
     inputs:
       khiops-revision:
         type: string
-        default: 11.0.0-b.0
+        default: 11.0.0-rc.1
         description: Khiops Revision
       image-increment:
         type: number
@@ -53,7 +53,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        khiopsdev-os: [ubuntu22.04, rocky8, rocky9]
+        khiopsdev-os: [ubuntu22.04, rocky8, rocky9, debian13]
     permissions:
       packages: write  # to write in the Github package registry
     steps:

.github/workflows/pip.yml

@@ -12,7 +12,7 @@ on:
         default: 11.0.0
         description: khiops-samples repo revision
       image-tag:
-        default: 11.0.0-b.0.0
+        default: 11.0.0-rc.1.0
         description: Development Docker Image Tag
       pypi-target:
         type: choice
@@ -65,13 +65,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        container: [ubuntu22.04, rocky9]
+        container: [ubuntu22.04, rocky9, debian13]
     container:
       # 'latest' default image tag cannot be set as an environment variable,
       # because the `env` context is only accessible at the step level;
       # hence, it is hard-coded
       image: |-
-        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || '11.0.0-b.0.0' }}
+        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || '11.0.0-rc.1.0' }}
     steps:
       - name: Set parameters as env
         run: |
@@ -89,28 +89,45 @@ jobs:
         with:
           name: pip-package
       - name: Install package
+        shell: bash
         run: |
           # Allow Pip to write to its cache
           mkdir -p /github/home/.cache/pip
           chown -R $(whoami) /github/home/.cache/pip
           # Install the Khiops Python library
+
+          # A virtual env is mandatory under debian
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then
+            python -m venv khiops-debian-venv
+            source khiops-debian-venv/bin/activate                      
+          fi
           pip install --upgrade pip
           pip install $(ls khiops*.tar.gz)
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            deactivate                      
+          fi
       - name: Run tests
         env:
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
           # Force > 2 CPU cores to launch mpiexec
           KHIOPS_PROC_NUMBER: 4
-          # Oversubscribe for MPI 4.x
+          # Oversubscribe for Open MPI 4.x
           rmaps_base_oversubscribe: true
-          # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
+          # Oversubscribe for Open MPI >= 5
+          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
+        shell: bash
         run: |-
           # Make sure MPI support is not loaded through env modules
           # Note: As the Docker container's shell is non-interactive, environment
           # modules are currently not initializing the shell anyway
           if [ -n "$MODULESHOME" ]; then module unload mpi; fi
 
+          # A virtual env is mandatory under debian
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            source khiops-debian-venv/bin/activate                      
+          fi
+
           # Print khiops installation status
           kh-status
 
@@ -124,6 +141,9 @@ jobs:
           # The MPI command is not always named mpiexec, but can be orterun etc
           # (as given by khiops_env)
           kh-status | grep "MPI command" | grep -vwq "<empty>"
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            deactivate                      
+          fi
       - name: Test package / Git tag version coherence
         shell: bash
         if: github.ref_type == 'tag'

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ on:
         default: 11.0.0
         description: Git Tag/Branch/Commit for the khiops-samples Repo
       image-tag:
-        default: 11.0.0-b.0.1
+        default: 11.0.0-rc.1.0
         description: Development Docker Image Tag
       khiops-desktop-revision:
         default: 11.0.0-b.0
@@ -37,13 +37,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
     container:
       # 'latest' default image tag cannot be set as an environment variable,
       # because the `env` context is only accessible at the step level;
       # hence, it is hard-coded
       image: |-
-        ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:${{ inputs.image-tag || '11.0.0-b.0.1' }}
+        ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:${{ inputs.image-tag || '11.0.0-rc.1.0' }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
@@ -175,10 +175,11 @@ jobs:
           KHIOPS_RUNNER_SERVICE_PATH: /scripts/run_service.sh
           # Force > 2 CPU cores to launch mpiexec
           KHIOPS_PROC_NUMBER: 4
-          # Oversubscribe for MPI 4.x
+          # Oversubscribe for Open MPI 4.x
           rmaps_base_oversubscribe: true
-          # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
+          # Oversubscribe for Open MPI >= 5
+          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
           # Var for tests with S3
           no_proxy: localhost
         run: |
@@ -326,14 +327,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        container: [ubuntu22.04, rocky8, rocky9]
+        container: [ubuntu22.04, rocky8, rocky9, debian13]
     runs-on: ubuntu-latest
     container:
       # 'latest' default image tag cannot be set as an environment variable,
       # because the `env` context is only accessible at the step level;
       # hence, it is hard-coded
       image: |-
-        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || '11.0.0-b.0.1' }}
+        ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || '11.0.0-rc.1.0' }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
@@ -357,28 +358,49 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           path: khiops-samples
       - name: Install khiops-python dev dependencies
+        shell: bash
         run: |
           # The following git command is required,
           # as the Git repository is in a directory the current user does not own,
           # Python versioneer fails to compute the current version correctly otherwise
           git config --global --add safe.directory $(realpath .)
+
+          # A virtual env is mandatory under debian
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then
+            python -m venv khiops-debian-venv
+            source khiops-debian-venv/bin/activate                      
+          fi
+
           # Install tomli for Python < 3.11
           pip install tomli
           python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" > requires.txt
           pip install `cat requires.txt`
           rm -f requires.txt
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            deactivate                      
+          fi
       - name: Setup and Install Test Requirements
-        run: |
+        shell: bash
+        run: |-
+          # A virtual env is mandatory under debian
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            source khiops-debian-venv/bin/activate                      
+          fi
           pip install -r test-requirements.txt
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            deactivate                      
+          fi
       - name: Test Khiops Integration
         env:
           # Force > 2 CPU cores to launch mpiexec
           KHIOPS_PROC_NUMBER: 4
           KHIOPS_SAMPLES_DIR: ${{ github.workspace }}/khiops-samples
-          # Oversubscribe for MPI 4.x
+          # Oversubscribe for Open MPI 4.x
           rmaps_base_oversubscribe: true
-          # Oversubscribe for MPI > 4.x
           OMPI_MCA_rmaps_base_oversubscribe: true
+          # Oversubscribe for Open MPI >= 5
+          PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
+        shell: bash
         run: |-
           # Reset the default 'exit-on-error' mode of the bash shell in Github actions
           # so that the return code can be evaluated if needed
@@ -389,6 +411,11 @@ jobs:
           # modules are currently not initializing the shell anyway
           if [ -n "$MODULESHOME" ]; then module unload mpi; fi
 
+          # A virtual env is mandatory under debian
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            source khiops-debian-venv/bin/activate                      
+          fi
+
           # Print status
           python -c \
             "import sys; import khiops.core as kh; return_code = kh.get_runner().print_status(); sys.exit(return_code)"
@@ -404,3 +431,6 @@ jobs:
 
           # Execute Khiops sample (train and deploy model), which uses MPI
           python -m khiops.samples.samples -i deploy_model -e
+          if [[ "${{ matrix.container }}" == "debian13" ]]; then            
+            deactivate                      
+          fi

khiops/core/internals/runner.py

@@ -327,6 +327,22 @@ def _get_current_library_installer():
         return "unknown"
 
 
+def _build_khiops_process_environment():
+    """Build a specific environment used for the execution of khiops in a process
+
+    This environment can be modified freely without interfering
+    with the global one.
+    """
+    khiops_env = os.environ.copy()
+
+    # Ensure HOME is always set for OpenMPI 5+
+    # (using KHIOPS_MPI_HOME if it exists)
+    khiops_env["HOME"] = os.path.pathsep.join(
+        [khiops_env.get("KHIOPS_MPI_HOME", ""), khiops_env.get("HOME", "")]
+    )
+    return khiops_env
+
+
 class KhiopsRunner(ABC):
     """Abstract Khiops Python runner to be re-implemented"""
 
@@ -994,8 +1010,14 @@ def _initialize_khiops_environment(self):
                     var_value = ""
                 else:
                     continue
+
+                # Special var to export in order
+                # to prepend to or to set to HOME for OpenMPI 5+
+                # when running khiops core
+                if var_name == "KHIOPS_MPI_HOME":
+                    os.environ["KHIOPS_MPI_HOME"] = var_value
                 # Set paths to Khiops binaries
-                if var_name == "KHIOPS_PATH":
+                elif var_name == "KHIOPS_PATH":
                     self.khiops_path = var_value
                     os.environ["KHIOPS_PATH"] = var_value
                 elif var_name == "KHIOPS_COCLUSTERING_PATH":
@@ -1443,6 +1465,11 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False):
                     khiops_call += f" {quote}{arg}{quote}"
             print(f"Khiops execution call: {khiops_call}")
 
+        # Build custom Khiops process environment
+        # which makes sure HOME is defined and set
+        # according to khiops_env's KHIOPS_MPI_HOME
+        khiops_env = _build_khiops_process_environment()
+
         # Execute the process
         with subprocess.Popen(
             khiops_process_args,
@@ -1452,6 +1479,7 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False):
             encoding="utf8",
             errors="replace",
             universal_newlines=True,
+            env=khiops_env,
         ) as khiops_process:
             stdout, stderr = khiops_process.communicate()