Type conversions for Python Series constructor (#1737)

* Accept std::filesystem::path as path argument in constructor

* Same trick for JSON config

* Fix Nompi runs

* Better distinguish JSON objects and MPI communicators

* WIP: examples, testing, fixes

* Same trick for Dataset() constructor

* CI fixes

* Fallback implementation for old compilers

* Fix non-Python builds

* Fixes

* Ehh dont use the std::filesystem implementation by default

* Type Check for filepath argument

* Change CI options

* Document Series constructor, add filepath option for config

Also test

* Document build option

* Move helper to auxiliary header

* Fix after rebase

* Document helper

Author: franzpoeschel

.github/workflows/linux.yml

@@ -266,10 +266,11 @@ jobs:
       env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig}
       run: |
         share/openPMD/download_samples.sh build
-        cmake -S . -B build \
-          -DopenPMD_USE_PYTHON=ON \
-          -DopenPMD_USE_MPI=ON    \
-          -DopenPMD_USE_HDF5=ON   \
+        cmake -S . -B build                  \
+          -DopenPMD_USE_PYTHON=ON            \
+          -DopenPMD_USE_MPI=ON               \
+          -DopenPMD_USE_HDF5=ON              \
+          -DopenPMD_USE_FILESYSTEM_HEADER=ON \
           -DopenPMD_USE_INVASIVE_TESTS=ON
         cmake --build build --parallel 4
         ctest --test-dir build --output-on-failure
@@ -290,11 +291,12 @@ jobs:
       env: {CXXFLAGS: -Werror}
       run: |
         share/openPMD/download_samples.sh build
-        cmake -S . -B build \
-          -DopenPMD_USE_PYTHON=ON \
-          -DopenPMD_USE_MPI=OFF   \
-          -DopenPMD_USE_HDF5=ON   \
-          -DopenPMD_USE_INVASIVE_TESTS=ON \
+        cmake -S . -B build                  \
+          -DopenPMD_USE_PYTHON=ON            \
+          -DopenPMD_USE_MPI=OFF              \
+          -DopenPMD_USE_HDF5=ON              \
+          -DopenPMD_USE_INVASIVE_TESTS=ON    \
+          -DopenPMD_USE_FILESYSTEM_HEADER=ON \
           -DPython_EXECUTABLE=$(which python3.10)
         cmake --build build --parallel 4
         ctest --test-dir build --output-on-failure
@@ -317,13 +319,14 @@ jobs:
       env: {CXXFLAGS: -Werror}
       run: |
         share/openPMD/download_samples.sh build
-        cmake -S . -B build \
+        cmake -S . -B build                                \
           -DCMAKE_CXX_FLAGS="-Wno-error=stringop-overread" \
-          -DCMAKE_C_FLAGS="-Wno-error=stringop-overread" \
-          -DopenPMD_USE_PYTHON=ON \
-          -DopenPMD_USE_MPI=ON    \
-          -DopenPMD_USE_HDF5=ON   \
-          -DopenPMD_USE_ADIOS2=ON \
+          -DCMAKE_C_FLAGS="-Wno-error=stringop-overread"   \
+          -DopenPMD_USE_PYTHON=ON                          \
+          -DopenPMD_USE_MPI=ON                             \
+          -DopenPMD_USE_HDF5=ON                            \
+          -DopenPMD_USE_ADIOS2=ON                          \
+          -DopenPMD_USE_FILESYSTEM_HEADER=ON               \
           -DopenPMD_USE_INVASIVE_TESTS=ON
         cmake --build build --parallel 4
         ctest --test-dir build --output-on-failure

CMakeLists.txt

@@ -151,6 +151,8 @@ option(openPMD_USE_INTERNAL_TOML11   "Use internally shipped toml11"        ${op
 
 option(openPMD_USE_INVASIVE_TESTS "Enable unit tests that modify source code" OFF)
 option(openPMD_USE_VERIFY "Enable internal VERIFY (assert) macro independent of build type" ON)
+option(openPMD_USE_FILESYSTEM_HEADER "Enable filesystem header. May be disabled for old compiler versions." OFF)
+mark_as_advanced(openPMD_USE_FILESYSTEM_HEADER)
 
 set(CMAKE_CONFIGURATION_TYPES "Release;Debug;MinSizeRel;RelWithDebInfo")
 if(NOT CMAKE_BUILD_TYPE)
@@ -552,6 +554,7 @@ endif()
 if(openPMD_HAVE_PYTHON)
     add_library(openPMD.py MODULE
         src/binding/python/openPMD.cpp
+        src/binding/python/auxiliary.cpp
         src/binding/python/Access.cpp
         src/binding/python/Attributable.cpp
         src/binding/python/BaseRecordComponent.cpp
@@ -732,6 +735,18 @@ if(openPMD_USE_INVASIVE_TESTS)
     target_compile_definitions(openPMD PRIVATE openPMD_USE_INVASIVE_TESTS=1)
 endif()
 
+function(set_filesystem_header_for_target target)
+    if(openPMD_USE_FILESYSTEM_HEADER)
+        target_compile_definitions(${target} PRIVATE openPMD_USE_FILESYSTEM_HEADER=1)
+    else()
+        target_compile_definitions(${target} PRIVATE openPMD_USE_FILESYSTEM_HEADER=0)
+    endif()
+endfunction()
+set_filesystem_header_for_target(openPMD)
+if(openPMD_HAVE_PYTHON)
+    set_filesystem_header_for_target(openPMD.py)
+endif()
+
 if(openPMD_BUILD_TESTING)
     # compile Catch2 implementation part separately
     add_library(CatchRunner ${_openpmd_lib_type}

docs/source/dev/buildoptions.rst

@@ -9,24 +9,27 @@ Variants
 The following options can be added to the ``cmake`` call to control features.
 CMake controls options with prefixed ``-D``, e.g. ``-DopenPMD_USE_MPI=OFF``:
 
-============================== =============== ========================================================================
-CMake Option                   Values          Description
-============================== =============== ========================================================================
-``openPMD_USE_MPI``            **AUTO**/ON/OFF Parallel, Multi-Node I/O for clusters
-``openPMD_USE_HDF5``           **AUTO**/ON/OFF HDF5 backend (``.h5`` files)
-``openPMD_USE_ADIOS2``         **AUTO**/ON/OFF ADIOS2 backend (``.bp`` files in BP3, BP4 or higher)
-``openPMD_USE_PYTHON``         **AUTO**/ON/OFF Enable Python bindings
-``openPMD_USE_INVASIVE_TESTS`` ON/**OFF**      Enable unit tests that modify source code :sup:`1`
-``openPMD_USE_VERIFY``         **ON**/OFF      Enable internal VERIFY (assert) macro independent of build type :sup:`2`
-``openPMD_INSTALL``            **ON**/OFF      Add installation targets
-``openPMD_INSTALL_RPATH``      **ON**/OFF      Add RPATHs to installed binaries
-``Python_EXECUTABLE``          (newest found)  Path to Python executable
-============================== =============== ========================================================================
+================================= =============== ===============================================================================
+CMake Option                      Values          Description
+================================= =============== ===============================================================================
+``openPMD_USE_MPI``               **AUTO**/ON/OFF Parallel, Multi-Node I/O for clusters
+``openPMD_USE_HDF5``              **AUTO**/ON/OFF HDF5 backend (``.h5`` files)
+``openPMD_USE_ADIOS2``            **AUTO**/ON/OFF ADIOS2 backend (``.bp`` files in BP3, BP4 or higher)
+``openPMD_USE_PYTHON``            **AUTO**/ON/OFF Enable Python bindings
+``openPMD_USE_INVASIVE_TESTS``    ON/**OFF**      Enable unit tests that modify source code :sup:`1`
+``openPMD_USE_VERIFY``            **ON**/OFF      Enable internal VERIFY (assert) macro independent of build type :sup:`2`
+``openPMD_INSTALL``               **ON**/OFF      Add installation targets
+``openPMD_INSTALL_RPATH``         **ON**/OFF      Add RPATHs to installed binaries
+``Python_EXECUTABLE``             (newest found)  Path to Python executable
+``openPMD_USE_FILESYSTEM_HEADER`` ON/**OFF**      In-/Exclude optional features implemented with ``<filesystem>`` header :sup:`3`
+================================= =============== ===============================================================================
 
 :sup:`1` e.g. changes C++ visibility keywords, breaks MSVC
 
 :sup:`2` this includes most pre-/post-condition checks, disabling without specific cause is highly discouraged
 
+:sup:`3` currently only used for supporting ``pathlib.Path``-type arguments in the Python API; a manual fallback implementation is used otherwise
+
 
 Shared or Static
 ----------------

examples/10_streaming_read.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-import json
 import sys
 
 import openpmd_api as io
@@ -18,8 +17,7 @@
         print("SST engine not available in ADIOS2.")
         sys.exit(0)
 
-    series = io.Series("simData.sst", io.Access_Type.read_linear,
-                       json.dumps(config))
+    series = io.Series("simData.sst", io.Access_Type.read_linear, config)
 
     # Read all available iterations and print electron position data.
     # Direct access to iterations is possible via `series.iterations`.

examples/10_streaming_write.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-import json
 import sys
 
 import numpy as np
@@ -21,8 +20,7 @@
 
     # create a series and specify some global metadata
     # change the file extension to .json, .h5 or .bp for regular file writing
-    series = io.Series("simData.sst", io.Access_Type.create,
-                       json.dumps(config))
+    series = io.Series("simData.sst", io.Access_Type.create, config)
     series.set_author("Franz Poeschel <f.poeschel@hzdr.de>")
     series.set_software("openPMD-api-python-examples")
 

examples/13_write_dynamic_configuration.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-import json
-
 import numpy as np
 import openpmd_api as io
 
@@ -19,6 +17,9 @@
 # Alternatively, the location of a JSON/TOML-file on the filesystem can
 # be passed by adding an at-sign `@` in front of the path
 # The format will then be recognized by filename extension, i.e. .json or .toml
+# In Python, normal Python dictionaries can also be used which will then be
+# converted via `json.dumps()` in the Series constructor
+# (see below for an example in terms of the Dataset constructor)
 
 backend = "adios2"
 iteration_encoding = "group_based"
@@ -105,9 +106,6 @@ def main():
                 'dataset': {
                     'operators': []
                 }
-            },
-            'adios1': {
-                'dataset': {}
             }
         }
         config['adios2']['dataset'] = {
@@ -118,9 +116,6 @@ def main():
                 }
             }]
         }
-        config['adios1']['dataset'] = {
-            'transform': 'blosc:compressor=zlib,shuffle=bit,lvl=1;nometa'
-        }
 
         temperature = iteration.meshes["temperature"]
         temperature.unit_dimension = {io.Unit_Dimension.theta: 1.0}
@@ -129,8 +124,7 @@ def main():
         # temperature has no x,y,z components, so skip the last layer:
         temperature_dataset = temperature
         # let's say we are in a 3x3 mesh
-        dataset = io.Dataset(np.dtype("double"), [3, 3])
-        dataset.options = json.dumps(config)
+        dataset = io.Dataset(np.dtype("double"), [3, 3], config)
         temperature_dataset.reset_dataset(dataset)
         # temperature is constant
         local_data = np.arange(i * 9, (i + 1) * 9, dtype=np.dtype("double"))

examples/7_extended_write_serial.py

@@ -6,8 +6,6 @@
 Authors: Axel Huebl, Fabian Koller
 License: LGPLv3+
 """
-import json
-
 import numpy as np
 from openpmd_api import (Access, Dataset, Mesh_Record_Component, Series,
                          Unit_Dimension)
@@ -104,7 +102,6 @@
     # before storing record data, you must specify the dataset once per
     # component this describes the datatype and shape of data as it should be
     # written to disk
-    d = Dataset(partial_mesh.dtype, extent=[2, 5])
     dataset_config = {
         "adios2": {
             "dataset": {
@@ -117,7 +114,7 @@
             }
         }
     }
-    d.options = json.dumps(dataset_config)
+    d = Dataset(partial_mesh.dtype, extent=[2, 5], options=dataset_config)
     mesh["x"].reset_dataset(d)
 
     electrons = cur_it.particles["electrons"]

examples/9_particle_write_serial.py

@@ -6,19 +6,16 @@
 Authors: Axel Huebl
 License: LGPLv3+
 """
-import numpy as np
-from openpmd_api import (Access, Dataset, Mesh_Record_Component, Series,
-                         Unit_Dimension)
-
-SCALAR = Mesh_Record_Component.SCALAR
+from pathlib import Path
 
+import numpy as np
+from openpmd_api import Access, Dataset, Series, Unit_Dimension
 
 if __name__ == "__main__":
     # open file for writing
-    f = Series(
-        "../samples/9_particle_write_serial_py.h5",
-        Access.create
-    )
+    samples = Path("../samples")
+    filename = "9_particle_write_serial_py.h5"
+    f = Series(samples / filename, Access.create)
 
     # all required openPMD attributes will be set to reasonable default values
     # (all ones, all zeros, empty strings,...)

include/openPMD/binding/python/Common.hpp

@@ -26,6 +26,10 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 #include <pybind11/stl_bind.h>
+
+#if openPMD_USE_FILESYSTEM_HEADER
+#include <pybind11/stl/filesystem.h>
+#endif
 // not yet used:
 //   pybind11/functional.h  // for std::function
 

include/openPMD/binding/python/Mpi.hpp

@@ -43,7 +43,25 @@ struct openPMD_PyMPICommObject
 };
 using openPMD_PyMPIIntracommObject = openPMD_PyMPICommObject;
 
-inline std::variant<MPI_Comm, std::string>
+struct py_object_to_mpi_comm_error
+{
+    enum class error_type
+    {
+        invalid_data,
+        is_not_an_mpi_communicator
+    };
+    using error_type = error_type;
+
+    std::string error_msg;
+    error_type type;
+
+    operator std::string() const
+    {
+        return error_msg;
+    }
+};
+
+inline std::variant<MPI_Comm, py_object_to_mpi_comm_error>
 pythonObjectAsMpiComm(pybind11::object &comm)
 {
     namespace py = pybind11;
@@ -55,10 +73,13 @@ pythonObjectAsMpiComm(pybind11::object &comm)
     //! - installed: include/mpi4py/mpi4py.MPI_api.h
     // if( import_mpi4py() < 0 ) { here be dragons }
 
+    using e = py_object_to_mpi_comm_error;
+    using e_t = e::error_type;
+
     if (comm.ptr() == Py_None)
-        return {"MPI communicator cannot be None."};
+        return e{"MPI communicator cannot be None.", e_t::invalid_data};
     if (comm.ptr() == nullptr)
-        return {"MPI communicator is a nullptr."};
+        return e{"MPI communicator is a nullptr.", e_t::invalid_data};
 
     // check type string to see if this is mpi4py
     //   __str__ (pretty)
@@ -68,15 +89,18 @@ pythonObjectAsMpiComm(pybind11::object &comm)
     py::str const comm_pystr = py::repr(comm);
     std::string const comm_str = comm_pystr.cast<std::string>();
     if (comm_str.substr(0, 12) != std::string("<mpi4py.MPI."))
-        return {"comm is not an mpi4py communicator: " + comm_str};
+        return e{
+            "comm is not an mpi4py communicator: " + comm_str,
+            e_t::is_not_an_mpi_communicator};
     // only checks same layout, e.g. an `int` in `PyObject` could pass this
     if (!py::isinstance<py::class_<openPMD_PyMPIIntracommObject> >(
             comm.get_type()))
         // TODO add mpi4py version from above import check to error message
-        return {
+        return e{
             "comm has unexpected type layout in " + comm_str +
-            " (Mismatched MPI at compile vs. runtime? "
-            "Breaking mpi4py release?)"};
+                " (Mismatched MPI at compile vs. runtime? "
+                "Breaking mpi4py release?)",
+            e_t::invalid_data};
 
     // todo other possible implementations:
     // - pyMPI (inactive since 2008?): import mpi; mpi.WORLD
@@ -87,12 +111,13 @@ pythonObjectAsMpiComm(pybind11::object &comm)
         &((openPMD_PyMPIIntracommObject *)(comm.ptr()))->ob_mpi;
 
     if (PyErr_Occurred())
-        return {"MPI communicator access error."};
+        return e{"MPI communicator access error.", e_t::invalid_data};
     if (mpiCommPtr == nullptr)
     {
-        return {
+        return e{
             "MPI communicator cast failed. "
-            "(Mismatched MPI at compile vs. runtime?)"};
+            "(Mismatched MPI at compile vs. runtime?)",
+            e_t::invalid_data};
     }
     return {*mpiCommPtr};
 }