json::merge(): Two little fixes (#1333)

* Specify and test: JSON/TOML as input/output for json::merge

* Add Python binding

* Sort lines for the TOML equality check

* Fix NVCC TOML ios_base::openmode

* Revert "Fix NVCC TOML ios_base::openmode"

This reverts commit 22f0585.

* deactivate json_merging test for nvc++

Author: franzpoeschel

include/openPMD/auxiliary/JSON.hpp

@@ -28,16 +28,16 @@ namespace openPMD
 namespace json
 {
     /**
-     * @brief Merge two JSON datasets into one.
+     * @brief Merge two JSON/TOML datasets into one.
      *
      * Merging rules:
-     * 1. If both `defaultValue` and `overwrite` are JSON objects, then the
-     *    resulting JSON object will contain the union of both objects' keys.
-     *    If a key is specified in both objects, the values corresponding to the
-     *    key are merged recursively.
-     *    Keys that point to a null value after this procedure will be pruned.
-     * 2. In any other case, the JSON dataset `defaultValue` is replaced in its
-     *    entirety with the JSON dataset `overwrite`.
+     * 1. If both `defaultValue` and `overwrite` are JSON/TOML objects, then the
+     *    resulting JSON/TOML object will contain the union of both objects'
+     *    keys. If a key is specified in both objects, the values corresponding
+     *    to the key are merged recursively. Keys that point to a null value
+     *    after this procedure will be pruned.
+     * 2. In any other case, the JSON/TOML dataset `defaultValue` is replaced in
+     *    its entirety with the JSON/TOML dataset `overwrite`.
      *
      * Note that item 2 means that datasets of different type will replace each
      * other without error.
@@ -46,15 +46,18 @@ namespace json
      *
      * Possible use case:
      * An application uses openPMD-api and wants to do the following:
-     * 1. Set some default backend options as JSON parameters.
+     * 1. Set some default backend options as JSON/TOML parameters.
      * 2. Let its users specify custom backend options additionally.
      *
      * By using the json::merge() function, this application can then allow
      * users to overwrite default options, while keeping any other ones.
      *
-     * @param defaultValue
-     * @param overwrite
-     * @return std::string
+     * @param defaultValue A string containing either a JSON or a TOML dataset.
+     * @param overwrite A string containing either a JSON or TOML dataset (does
+     * not need to be the same as `defaultValue`).
+     * @return std::string The merged dataset, according to the above rules. If
+     * `defaultValue` was a JSON dataset, then as a JSON string, otherwise as a
+     * TOML string.
      */
     std::string
     merge(std::string const &defaultValue, std::string const &overwrite);

src/auxiliary/JSON.cpp

@@ -570,8 +570,21 @@ merge(nlohmann::json &defaultVal, nlohmann::json const &overwrite)
 
 std::string merge(std::string const &defaultValue, std::string const &overwrite)
 {
-    auto res = parseOptions(defaultValue, /* considerFiles = */ false).config;
+    auto [res, returnFormat] =
+        parseOptions(defaultValue, /* considerFiles = */ false);
     merge(res, parseOptions(overwrite, /* considerFiles = */ false).config);
-    return res.dump();
+    switch (returnFormat)
+    {
+    case SupportedLanguages::JSON:
+        return res.dump();
+        break;
+    case SupportedLanguages::TOML: {
+        auto asToml = json::jsonToToml(res);
+        std::stringstream sstream;
+        sstream << asToml;
+        return sstream.str();
+    }
+    }
+    throw std::runtime_error("Unreachable!");
 }
 } // namespace openPMD::json

src/binding/python/Series.cpp

@@ -23,6 +23,7 @@
 #include <pybind11/stl.h>
 
 #include "openPMD/Series.hpp"
+#include "openPMD/auxiliary/JSON.hpp"
 #include "openPMD/config.hpp"
 
 #if openPMD_HAVE_MPI
@@ -231,4 +232,43 @@ this method.
             "write_iterations",
             &Series::writeIterations,
             py::keep_alive<0, 1>());
+
+    m.def(
+        "merge_json",
+        &json::merge,
+        py::arg("default_value") = "{}",
+        py::arg("overwrite") = "{}",
+        R"END(
+Merge two JSON/TOML datasets into one.
+
+Merging rules:
+1. If both `defaultValue` and `overwrite` are JSON/TOML objects, then the
+resulting JSON/TOML object will contain the union of both objects'
+keys. If a key is specified in both objects, the values corresponding
+to the key are merged recursively. Keys that point to a null value
+after this procedure will be pruned.
+2. In any other case, the JSON/TOML dataset `defaultValue` is replaced in
+its entirety with the JSON/TOML dataset `overwrite`.
+
+Note that item 2 means that datasets of different type will replace each
+other without error.
+It also means that array types will replace each other without any notion
+of appending or merging.
+
+Possible use case:
+An application uses openPMD-api and wants to do the following:
+1. Set some default backend options as JSON/TOML parameters.
+2. Let its users specify custom backend options additionally.
+
+By using the json::merge() function, this application can then allow
+users to overwrite default options, while keeping any other ones.
+
+Parameters:
+* default_value: A string containing either a JSON or a TOML dataset.
+* overwrite:     A string containing either a JSON or TOML dataset (does
+                 not need to be the same as `defaultValue`).
+* returns:       The merged dataset, according to the above rules.
+                 If `defaultValue` was a JSON dataset, then as a JSON string,
+                 otherwise as a TOML string.
+        )END");
 }

test/JSONTest.cpp

@@ -5,8 +5,12 @@
 
 #include <catch2/catch.hpp>
 
+#include <algorithm>
 #include <fstream>
+#include <sstream>
+#include <string>
 #include <variant>
+#include <vector>
 
 using namespace openPMD;
 
@@ -122,6 +126,7 @@ TEST_CASE("json_parsing", "[auxiliary]")
     REQUIRE(jsonUpper.dump() == jsonLower.dump());
 }
 
+#if !__NVCOMPILER // see https://github.com/ToruNiina/toml11/issues/205
 TEST_CASE("json_merging", "auxiliary")
 {
     std::string defaultVal = R"END(
@@ -174,7 +179,48 @@ TEST_CASE("json_merging", "auxiliary")
     REQUIRE(
         json::merge(defaultVal, overwrite) ==
         json::parseOptions(expect, false).config.dump());
+
+    {
+        // The TOML library doesn't guarantee a specific order of output
+        // so we need to sort lines to compare with expected results
+        auto sort_lines = [](std::string const &s) -> std::vector<std::string> {
+            std::vector<std::string> v;
+            std::istringstream sstream(s);
+            for (std::string line; std::getline(sstream, line);
+                 line = std::string())
+            {
+                v.push_back(std::move(line));
+            }
+            std::sort(v.begin(), v.end());
+            return v;
+        };
+        std::string leftJson = R"({"left": "val"})";
+        std::string rightJson = R"({"right": "val"})";
+        std::string leftToml = R"(left = "val")";
+        std::string rightToml = R"(right = "val")";
+
+        std::string resJson =
+            nlohmann::json::parse(R"({"left": "val", "right": "val"})").dump();
+        std::vector<std::string> resToml = [&sort_lines]() {
+            constexpr char const *raw = R"(
+left = "val"
+right = "val"
+        )";
+            std::istringstream istream(
+                raw, std::ios_base::binary | std::ios_base::in);
+            toml::value tomlVal = toml::parse(istream);
+            std::stringstream sstream;
+            sstream << tomlVal;
+            return sort_lines(sstream.str());
+        }();
+
+        REQUIRE(json::merge(leftJson, rightJson) == resJson);
+        REQUIRE(json::merge(leftJson, rightToml) == resJson);
+        REQUIRE(sort_lines(json::merge(leftToml, rightJson)) == resToml);
+        REQUIRE(sort_lines(json::merge(leftToml, rightToml)) == resToml);
+    }
 }
+#endif
 
 /*
  * This tests two things about the /data/snapshot attribute: