Merge pull request #381 from KhiopsML/380-use-mpi-for-khiops-coclustering

Use mpi for khiops coclustering

Author: popescu-v

.gitignore

@@ -45,6 +45,7 @@ tests/resources/*/output_reports
 tests/resources/*/output_kdic
 tests/resources/dictionary/copy_output_kdic
 tests/resources/scenario_generation/api/*/output
+tests/resources/scenario_generation/data_path_deprecation/*/output
 tests/resources/scenario_generation/general_options/output
 tests/resources/scenario_generation/runner_version/output
 tests/resources/*/copy_output_kdic

khiops/core/internals/runner.py

@@ -1119,9 +1119,8 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False):
             )
 
         # Build command line arguments
-        # Nota: Khiops Coclustering is executed without MPI
         khiops_process_args = []
-        if tool_name == "khiops" and use_mpi:
+        if use_mpi:
             khiops_process_args += self._mpi_command_args
         khiops_process_args += [self._tool_path(tool_name)]
         if command_line_args:

tests/test_khiops_integrations.py

@@ -4,7 +4,7 @@
 # which is available at https://spdx.org/licenses/BSD-3-Clause-Clear.html or         #
 # see the "LICENSE.md" file for more details.                                        #
 ######################################################################################
-"""Tests for executing fit multiple times on multi-table data"""
+"""Various integration tests"""
 
 import os
 import platform
@@ -13,8 +13,10 @@
 import subprocess
 import tempfile
 import unittest
+from unittest.mock import MagicMock, patch
 
 import khiops.core as kh
+import khiops.core.internals.filesystems as fs
 from khiops.core.exceptions import KhiopsEnvironmentError
 from khiops.core.internals.runner import KhiopsLocalRunner
 from khiops.extras.docker import KhiopsDockerRunner
@@ -208,6 +210,67 @@ def test_runner_environment_initialization(self):
 
         self.assertEqual(env_khiops_api_mode, "true")
 
+    def test_khiops_and_khiops_coclustering_are_run_with_mpi(self):
+        """Test that MODL and MODL_Coclustering are run with MPI"""
+
+        # Get current runner
+        runner = kh.get_runner()
+
+        # Get path to the Iris dataset
+        iris_data_dir = fs.get_child_path(runner.samples_dir, "Iris")
+
+        # Create the subprocess.Popen mock
+        mock_popen = MagicMock()
+        mock_popen.return_value.__enter__.return_value.communicate.return_value = (
+            b"",
+            b"",
+        )
+        mock_popen.return_value.__enter__.return_value.returncode = 0
+
+        # Run Khiops through an API function, using the mocked Popen, to capture
+        # its arguments
+        with patch("subprocess.Popen", mock_popen):
+            kh.check_database(
+                fs.get_child_path(iris_data_dir, "Iris.kdic"),
+                "Iris",
+                fs.get_child_path(iris_data_dir, "Iris.txt"),
+            )
+
+        # Check that the mocked Popen call arguments list starts with the MPI
+        # arguments, followed by the Khiops command
+        expected_command_args = runner.mpi_command_args + [runner.khiops_path]
+        self.assertTrue(len(mock_popen.call_args.args) > 0)
+        self.assertTrue(len(mock_popen.call_args.args[0]) > len(expected_command_args))
+        self.assertEqual(
+            mock_popen.call_args.args[0][: len(expected_command_args)],
+            expected_command_args,
+        )
+
+        # Run Khiops Coclustering through an API function, using the mocked Popen
+        # to capture its arguments
+        # Nest context managers for Python 3.8 compatibility
+        with patch("subprocess.Popen", mock_popen):
+            with tempfile.TemporaryDirectory() as temp_dir:
+                kh.train_coclustering(
+                    fs.get_child_path(iris_data_dir, "Iris.kdic"),
+                    "Iris",
+                    fs.get_child_path(iris_data_dir, "Iris.txt"),
+                    ["SepalLength", "PetalLength"],
+                    fs.get_child_path(temp_dir, "IrisCoclusteringResults.khcj"),
+                )
+
+        # Check that the mocked Popen call arguments list starts with the MPI
+        # arguments, followed by the Khiops Coclustering command
+        expected_command_args = runner.mpi_command_args + [
+            runner.khiops_coclustering_path
+        ]
+        self.assertTrue(len(mock_popen.call_args.args) > 0)
+        self.assertTrue(len(mock_popen.call_args.args[0]) > len(expected_command_args))
+        self.assertEqual(
+            mock_popen.call_args.args[0][: len(expected_command_args)],
+            expected_command_args,
+        )
+
 
 class KhiopsMultitableFitTests(unittest.TestCase):
     """Test if Khiops estimator can be fitted on multi-table data"""