Dump coclustering JSON report keys in the expected order

popescu-v · popescu-v · commit 1e306b2af7dd · 2025-05-27T11:00:10.000+02:00
Thus, the `write_khiops_json_file` method can be used to dump the JSON
report with the keys exactly in the same order as expected by
MODL_Coclustering.

Thus, coclustering reports are now verbatim-identical to the reference
reports.
diff --git a/khiops/core/coclustering_results.py b/khiops/core/coclustering_results.py
@@ -77,6 +77,127 @@ class CoclusteringResults(KhiopsJSONObject):
         Coclustering modeling report.
     """
 
+    # Set coclustering report order key specification
+    # pylint: disable=line-too-long
+    json_key_sort_spec = {
+        "tool": None,
+        "version": None,
+        "shortDescription": None,
+        "coclusteringReport": {
+            "summary": {
+                "instances": None,
+                "cells": None,
+                "nullCost": None,
+                "cost": None,
+                "level": None,
+                "initialDimensions": None,
+                "frequencyVariable": None,
+                "dictionary": None,
+                "database": None,
+                "samplePercentage": None,
+                "samplingMode": None,
+                "selectionVariable": None,
+                "selectionValue": None,
+            },
+            "dimensionSummaries": [
+                {
+                    "name": None,
+                    "isVarPart": None,
+                    "type": None,
+                    "parts": None,
+                    "initialParts": None,
+                    "values": None,
+                    "interest": None,
+                    "description": None,
+                    "min": None,
+                    "max": None,
+                },
+            ],
+            "dimensionPartitions": [
+                {
+                    "name": None,
+                    "type": None,
+                    "innerVariables": {
+                        "dimensionSummaries": [
+                            {
+                                "name": None,
+                                "type": None,
+                                "parts": None,
+                                "initialParts": None,
+                                "values": None,
+                                "interest": None,
+                                "description": None,
+                                "min": None,
+                                "max": None,
+                            }
+                        ],
+                        "dimensionPartitions": [
+                            {
+                                "name": None,
+                                "type": None,
+                                "intervals": [
+                                    {
+                                        "cluster": None,
+                                        "bounds": None,
+                                    }
+                                ],
+                                "valueGroups": [
+                                    {
+                                        "cluster": None,
+                                        "values": None,
+                                        "valueFrequencies": None,
+                                    }
+                                ],
+                                "defaultGroupIndex": None,
+                            }
+                        ],
+                    },
+                    "intervals": [
+                        {
+                            "cluster": None,
+                            "bounds": None,
+                        }
+                    ],
+                    "valueGroups": [
+                        {
+                            "cluster": None,
+                            "values": None,
+                            "valueFrequencies": None,
+                            "valueTypicalities": None,
+                        }
+                    ],
+                    "defaultGroupIndex": None,
+                },
+            ],
+            "dimensionHierarchies": [
+                {
+                    "name": None,
+                    "type": None,
+                    "clusters": [
+                        {
+                            "cluster": None,
+                            "parentCluster": None,
+                            "frequency": None,
+                            "interest": None,
+                            "hierarchicalLevel": None,
+                            "rank": None,
+                            "hierarchicalRank": None,
+                            "isLeaf": None,
+                            "shortDescription": None,
+                            "description": None,
+                        }
+                    ],
+                }
+            ],
+            "cellPartIndexes": None,
+            "cellFrequencies": None,
+        },
+        "khiops_encoding": None,
+        "ansi_chars": None,
+        "colliding_utf8_chars": None,
+    }
+    # pylint: enable=line-too-long
+
     def __init__(self, json_data=None):
         """See class docstring"""
         # Initialize super class
diff --git a/khiops/core/internals/io.py b/khiops/core/internals/io.py
@@ -145,13 +145,31 @@ class KhiopsJSONObject:
     sub_tool : str, optional
         Identifies the tool that originated the JSON file. Used by tools of the Khiops
         family such as PataText or Enneade.
+    json_key_sort_spec : dict, optional
+        Dictionary that specifies the order of the keys in the Khiops JSON report.
+        Its values are `None`, except when they are dictionaries themselves.
+
+        .. note::
+            This is a class attribute that can be set in subclasses, to specify
+            a key order when serializing the report in a JSON file, via the
+            ``write_khiops_json_file`` method.
+
     json_data : dict
         Python dictionary extracted from the Khiops JSON report file.
         **Deprecated** will be removed in Khiops 12.
     """
 
+    # Set default JSON key sort specification attribute
+    # Can be set in classes that specialize this class
+    json_key_sort_spec = None
+
     def __init__(self, json_data=None):
         """See class docstring"""
+        # Check the type of the json_key_sort_spec class attribute
+        assert self.json_key_sort_spec is None or isinstance(
+            self.json_key_sort_spec, dict
+        ), type_error_message("key_sort_spec", self.json_key_sort_spec, dict)
+
         # Check the type of json_data
         if json_data is not None and not isinstance(json_data, dict):
             raise TypeError(type_error_message("json_data", json_data, dict))
@@ -261,17 +279,76 @@ def to_dict(self):
             report["subTool"] = self.sub_tool
         return report
 
-    def write_khiops_json_file(self, json_file_path):
+    def _json_key_sort_by_spec(self, jdict, key_sort_spec=None):
+        # json_key_sort_spec must be set before using this method
+        assert self.json_key_sort_spec is not None
+
+        # Handle the base case with non-None key_sort_spec
+        sorted_jdict = {}
+        if key_sort_spec is None:
+            key_sort_spec = self.json_key_sort_spec
+
+        # Iterate over the current fields and recurse if necessary
+        for spec_key, spec_value in key_sort_spec.items():
+            if not (spec_value is None or isinstance(spec_value, (dict, list))):
+                raise ValueError(
+                    type_error_message(
+                        "specification value",
+                        spec_value,
+                        "'None' or dict or list",
+                    )
+                )
+            if spec_key in jdict:
+                json_value = jdict[spec_key]
+
+                # If json_value is not a dict, then:
+                # - if not list-like, then add it as such to the output dict
+                # - else, iterate on the list-like value
+                # else, recurse on the dict structure
+                if not isinstance(json_value, dict):
+                    if not isinstance(json_value, list):
+                        sorted_jdict[spec_key] = json_value
+                    else:
+                        sorted_jdict[spec_key] = []
+                        for json_el in json_value:
+                            if not isinstance(json_el, dict):
+                                sorted_jdict[spec_key].append(json_el)
+                            else:
+                                if isinstance(spec_value, list):
+                                    sorted_jdict[spec_key].append(
+                                        self._json_key_sort_by_spec(
+                                            json_el, key_sort_spec=spec_value[0]
+                                        )
+                                    )
+                else:
+                    sorted_jdict[spec_key] = self._json_key_sort_by_spec(
+                        json_value, key_sort_spec=spec_value
+                    )
+        return sorted_jdict
+
+    def write_khiops_json_file(self, json_file_path, _ensure_ascii=False):
         """Write the JSON data of the object to a Khiops JSON file
 
+        The JSON keys are sorted according to the
+        ``KhiopsJSONObject.json_key_sort_spec`` class attribute, if set.
+        Otherwise, the JSON keys are not sorted.
+
         Parameters
         ----------
         json_file_path : str
             Path to the Khiops JSON file.
+        _ensure_ascii : bool, default False
+            If True, then non-ASCII characters in the report are escaped. Otherwise,
+            they are dumped as-is.
         """
         # Serialize JSON data to string
         # Do not escape non-ASCII Unicode characters
-        json_string = json.dumps(self.to_dict(), ensure_ascii=False)
+        json_dict = self.to_dict()
+        if self.json_key_sort_spec is not None:
+            json_dict = self._json_key_sort_by_spec(json_dict)
+            json_string = json.dumps(json_dict, indent=4, ensure_ascii=_ensure_ascii)
+        else:
+            json_string = json.dumps(json_dict, indent=4, ensure_ascii=_ensure_ascii)
         with io.BytesIO() as json_stream:
             writer = self.create_output_file_writer(json_stream)
             writer.write(json_string)
diff --git a/tests/test_core.py b/tests/test_core.py
@@ -10,6 +10,7 @@
 import json
 import os
 import shutil
+import tempfile
 import textwrap
 import unittest
 import warnings
@@ -33,7 +34,45 @@
 class KhiopsCoreIOTests(unittest.TestCase):
     """Tests the reading/writing of files for the core module classes/functions"""
 
-    def _assert_report_is_dumped_to_correct_json(self, report, ref_json_report):
+    def _assert_coclustering_report_is_written_to_sorted_json_file(
+        self, cc_report, ref_json_report
+    ):
+        # Write the coclustering report to a JSON file, sorted according to
+        # the spec defined in the CoclusteringResults class
+        # Set _ensure_ascii, as non-ASCII characters are escaped in the reference
+        # reports
+        tmp_dir = tempfile.mkdtemp()
+        output_report = os.path.join(tmp_dir, "TestCoclustering.khcj")
+        cc_report.write_khiops_json_file(output_report, _ensure_ascii=True)
+
+        # Load JSON Khiops reports into Python dictionaries
+        with open(ref_json_report, encoding="utf-8") as ref_json_file:
+            ref_json = json.load(ref_json_file)
+        with open(output_report, encoding="utf-8") as output_json_file:
+            output_json = json.load(output_json_file)
+
+        # Dump reports with consistent indentation
+        ref_json_string = json.dumps(ref_json, indent=4)
+        output_json_string = json.dumps(output_json, indent=4)
+
+        # Succeed if the dumped reports are equal
+        if output_json_string == ref_json_string:
+            shutil.rmtree(tmp_dir)
+            return
+
+        # On failure print the differences
+        output_json_lines = output_json_string.splitlines(keepends=True)
+        ref_json_lines = ref_json_string.splitlines(keepends=True)
+        out_ref_diff = "".join(unified_diff(ref_json_lines, output_json_lines))
+        if out_ref_diff:
+            self.fail(
+                "CoclusteringResults JSON dump differs from reference "
+                f"'{ref_json_report}':\n{out_ref_diff}"
+            )
+
+    def _assert_analysis_report_is_dumped_to_correct_json(
+        self, report, ref_json_report
+    ):
         # Dump the report as JSON (4-space indented and keys sorted in
         # lexicographic order)
         output_json = report.to_dict()
@@ -54,11 +93,8 @@ def _assert_report_is_dumped_to_correct_json(self, report, ref_json_report):
         ref_json_lines = ref_json_string.splitlines(keepends=True)
         out_ref_diff = "".join(unified_diff(ref_json_lines, output_json_lines))
         if out_ref_diff:
-            report_type = (
-                "Analysis" if ref_json_report.endswith(".khj") else "Coclustering"
-            )
             self.fail(
-                f"{report_type}Results JSON dump differs from reference "
+                f"AnalysisResults JSON dump differs from reference "
                 f"'{ref_json_report}':\n{out_ref_diff}"
             )
 
@@ -111,12 +147,12 @@ def test_analysis_results(self):
                 elif report in reports_warn:
                     with self.assertWarns(UserWarning):
                         results = kh.read_analysis_results_file(ref_json_report)
-                        self._assert_report_is_dumped_to_correct_json(
+                        self._assert_analysis_report_is_dumped_to_correct_json(
                             results, ref_json_report
                         )
                 else:
                     results = kh.read_analysis_results_file(ref_json_report)
-                    self._assert_report_is_dumped_to_correct_json(
+                    self._assert_analysis_report_is_dumped_to_correct_json(
                         results, ref_json_report
                     )
 
@@ -152,7 +188,9 @@ def test_coclustering_results(self):
                         results = kh.read_coclustering_results_file(ref_json_report)
                 else:
                     results = kh.read_coclustering_results_file(ref_json_report)
-                self._assert_report_is_dumped_to_correct_json(results, ref_json_report)
+                self._assert_coclustering_report_is_written_to_sorted_json_file(
+                    results, ref_json_report
+                )
 
     def test_binary_dictionary_domain(self):
         """Test binary dictionary write"""
