Initial support for HDF5 subfiling (#1580)

* Initial support for HDF5 subfiling

* Add JSON/TOML parameterization

* Turn errors into warnings when subfiling is unavailable

* Add documentation

Author: franzpoeschel

docs/source/backends/hdf5.rst

@@ -15,6 +15,15 @@ I/O Method
 
 HDF5 internally either writes serially, via ``POSIX`` on Unix systems, or parallel to a single logical file via MPI-I/O.
 
+Virtual File Drivers
+********************
+
+Rudimentary support for HDF5 VFDs (`virtual file driver <https://www.hdfgroup.org/wp-content/uploads/2021/10/HDF5-VFD-Plugins-HUG.pdf>`_) is available (currently only the *subfiling* VFD).
+Note that the subfiling VFD needs to be enabled explicitly when configuring HDF5 and threaded MPI must be used.
+
+Virtual file drivers are configured via JSON/TOML.
+Refer to the page on `JSON/TOML configuration <backendconfig-hdf5>`_ for further details.
+
 
 Backend-Specific Controls
 -------------------------

docs/source/details/backendconfig.rst

@@ -190,6 +190,18 @@ Explanation of the single keys:
   The default is ``"auto"`` for a heuristic.
   ``"none"`` can be used to disable chunking.
   Chunking generally improves performance and only needs to be disabled in corner-cases, e.g. when heavily relying on independent, parallel I/O that non-collectively declares data records.
+* ``hdf5.vfd.type`` selects the HDF5 virtual file driver.
+  Currently available are:
+
+  * ``"default"``: Equivalent to specifying nothing.
+  * ``subfiling"``: Use the `subfiling VFD <https://www.hdfgroup.org/wp-content/uploads/2022/09/HDF5-Subfiling-VFD.pdf>`_.
+    Note that the subfiling VFD needs to be enabled explicitly when configuring HDF5 and threaded MPI must be used.
+    When using this VFD, the options described below are additionally available.
+    They correspond with the field entries of ``H5FD_subfiling_params_t``, refer to the HDF5 documentation for their detailed meanings.
+
+    * ``hdf5.vfd.ioc_selection``: Must be one of ``["one_per_node", "every_nth_rank", "with_config", "total"]``
+    * ``hdf5.vfd.stripe_size``: Must be an integer
+    * ``hdf5.vfd.stripe_count``: Must be an integer
 
 .. _backendconfig-other:
 

docs/source/details/hdf5.json

@@ -2,6 +2,12 @@
   "hdf5": {
     "dataset": {
       "chunks": "auto"
+    },
+    "vfd": {
+      "type": "subfiling",
+      "ioc_selection": "every_nth_rank",
+      "stripe_size": 33554432,
+      "stripe_count": -1
     }
   }
 }

examples/5_write_parallel.cpp

@@ -31,7 +31,8 @@ using namespace openPMD;
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
 
     int mpi_size;
     int mpi_rank;
@@ -47,9 +48,20 @@ int main(int argc, char *argv[])
         cout << "Set up a 2D array with 10x300 elements per MPI rank ("
              << mpi_size << "x) that will be written to disk\n";
 
+    std::string subfiling_config = R"(
+[hdf5.vfd]
+type = "subfiling"
+ioc_selection = "every_nth_rank"
+stripe_size = 33554432
+stripe_count = -1
+        )";
+
     // open file for writing
     Series series = Series(
-        "../samples/5_parallel_write.h5", Access::CREATE, MPI_COMM_WORLD);
+        "../samples/5_parallel_write.h5",
+        Access::CREATE,
+        MPI_COMM_WORLD,
+        subfiling_config);
     if (0 == mpi_rank)
         cout << "Created an empty series in parallel with " << mpi_size
              << " MPI ranks\n";

include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp

@@ -38,7 +38,10 @@ namespace openPMD
 class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
 {
 public:
-    HDF5IOHandlerImpl(AbstractIOHandler *, json::TracingJSON config);
+    HDF5IOHandlerImpl(
+        AbstractIOHandler *,
+        json::TracingJSON config,
+        bool do_warn_unused_params = true);
     ~HDF5IOHandlerImpl() override;
 
     void
@@ -114,8 +117,9 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
     std::optional<MPI_Comm> m_communicator;
 #endif
 
-private:
     json::TracingJSON m_config;
+
+private:
     std::string m_chunks = "auto";
     struct File
     {

src/IO/HDF5/HDF5IOHandler.cpp

@@ -66,7 +66,9 @@ namespace openPMD
 #endif
 
 HDF5IOHandlerImpl::HDF5IOHandlerImpl(
-    AbstractIOHandler *handler, json::TracingJSON config)
+    AbstractIOHandler *handler,
+    json::TracingJSON config,
+    bool do_warn_unused_params)
     : AbstractIOHandlerImpl(handler)
     , m_datasetTransferProperty{H5P_DEFAULT}
     , m_fileAccessProperty{H5P_DEFAULT}
@@ -167,23 +169,28 @@ HDF5IOHandlerImpl::HDF5IOHandlerImpl(
         }
 
         // unused params
-        auto shadow = m_config.invertShadow();
-        if (shadow.size() > 0)
+        if (do_warn_unused_params)
         {
-            switch (m_config.originallySpecifiedAs)
+            auto shadow = m_config.invertShadow();
+            if (shadow.size() > 0)
             {
-            case json::SupportedLanguages::JSON:
-                std::cerr << "Warning: parts of the backend configuration for "
-                             "HDF5 remain unused:\n"
-                          << shadow << std::endl;
-                break;
-            case json::SupportedLanguages::TOML: {
-                auto asToml = json::jsonToToml(shadow);
-                std::cerr << "Warning: parts of the backend configuration for "
-                             "HDF5 remain unused:\n"
-                          << asToml << std::endl;
-                break;
-            }
+                switch (m_config.originallySpecifiedAs)
+                {
+                case json::SupportedLanguages::JSON:
+                    std::cerr
+                        << "Warning: parts of the backend configuration for "
+                           "HDF5 remain unused:\n"
+                        << shadow << std::endl;
+                    break;
+                case json::SupportedLanguages::TOML: {
+                    auto asToml = json::jsonToToml(shadow);
+                    std::cerr
+                        << "Warning: parts of the backend configuration for "
+                           "HDF5 remain unused:\n"
+                        << asToml << std::endl;
+                    break;
+                }
+                }
             }
         }
     }

src/IO/HDF5/ParallelHDF5IOHandler.cpp

@@ -19,8 +19,17 @@
  * If not, see <http://www.gnu.org/licenses/>.
  */
 #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IO/HDF5/ParallelHDF5IOHandlerImpl.hpp"
 #include "openPMD/auxiliary/Environment.hpp"
+#include "openPMD/auxiliary/JSON_internal.hpp"
+#include "openPMD/auxiliary/StringManip.hpp"
+#include "openPMD/auxiliary/Variant.hpp"
+#include <type_traits>
+
+#ifdef H5_HAVE_SUBFILING_VFD
+#include <H5FDsubfiling.h>
+#endif
 
 #if openPMD_HAVE_MPI
 #include <mpi.h>
@@ -61,7 +70,7 @@ std::future<void> ParallelHDF5IOHandler::flush(internal::ParsedFlushParams &)
 
 ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl(
     AbstractIOHandler *handler, MPI_Comm comm, json::TracingJSON config)
-    : HDF5IOHandlerImpl{handler, std::move(config)}
+    : HDF5IOHandlerImpl{handler, std::move(config), /* do_warn_unused_params = */ false}
     , m_mpiComm{comm}
     , m_mpiInfo{MPI_INFO_NULL} /* MPI 3.0+: MPI_INFO_ENV */
 {
@@ -164,6 +173,173 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl(
     VERIFY(
         status >= 0,
         "[HDF5] Internal error: Failed to set HDF5 file access property");
+
+    if (!m_config.json().is_null() && m_config.json().contains("vfd"))
+    {
+        auto vfd_json_config = m_config["vfd"];
+        if (!vfd_json_config.json().contains("type"))
+        {
+            throw error::BackendConfigSchema(
+                {"hdf5", "vfd"},
+                "VFD configuration requires specifying the VFD type.");
+        }
+        std::string user_specified_type;
+        if (auto value =
+                json::asLowerCaseStringDynamic(vfd_json_config["type"].json());
+            value.has_value())
+        {
+            user_specified_type = *value;
+        }
+        else
+        {
+            throw error::BackendConfigSchema(
+                {"hdf5", "vfd", "type"}, "VFD type must be given as a string.");
+        }
+
+        if (user_specified_type == "default")
+        { /* no-op */
+        }
+        else if (user_specified_type == "subfiling")
+        {
+#ifdef H5_HAVE_SUBFILING_VFD
+            int thread_level = 0;
+            MPI_Query_thread(&thread_level);
+            if (thread_level >= MPI_THREAD_MULTIPLE)
+            {
+                H5FD_subfiling_config_t vfd_config;
+                // query default subfiling parameters
+                H5Pget_fapl_subfiling(m_fileAccessProperty, &vfd_config);
+
+                auto int_accessor =
+                    [&vfd_json_config](
+                        std::string const &key) -> std::optional<long long> {
+                    if (!vfd_json_config.json().contains(key))
+                    {
+                        return std::nullopt;
+                    }
+                    auto const &val = vfd_json_config[key].json();
+                    if (val.is_number_integer())
+                    {
+                        return val.get<long long>();
+                    }
+                    else
+                    {
+                        throw error::BackendConfigSchema(
+                            {"hdf5", "vfd", key},
+                            "Excpecting value of type integer.");
+                    }
+                };
+                auto string_accessor =
+                    [&vfd_json_config](
+                        std::string const &key) -> std::optional<std::string> {
+                    if (!vfd_json_config.json().contains(key))
+                    {
+                        return std::nullopt;
+                    }
+                    auto const &val = vfd_json_config[key].json();
+                    if (auto str_val = json::asLowerCaseStringDynamic(val);
+                        str_val.has_value())
+                    {
+                        return *str_val;
+                    }
+                    else
+                    {
+                        throw error::BackendConfigSchema(
+                            {"hdf5", "vfd", key},
+                            "Excpecting value of type string.");
+                    }
+                };
+
+                auto set_param = [](std::string const &key,
+                                    auto *target,
+                                    auto const &accessor) {
+                    if (auto val = accessor(key); val.has_value())
+                    {
+                        *target = static_cast<
+                            std::remove_reference_t<decltype(*target)>>(*val);
+                    }
+                };
+
+                set_param(
+                    "stripe_size",
+                    &vfd_config.shared_cfg.stripe_size,
+                    int_accessor);
+                set_param(
+                    "stripe_count",
+                    &vfd_config.shared_cfg.stripe_count,
+                    int_accessor);
+                std::optional<std::string> ioc_selection_raw;
+                set_param("ioc_selection", &ioc_selection_raw, string_accessor);
+
+                std::map<std::string, H5FD_subfiling_ioc_select_t> const
+                    ioc_selection_map{
+                        {"one_per_node", SELECT_IOC_ONE_PER_NODE},
+                        {"every_nth_rank", SELECT_IOC_EVERY_NTH_RANK},
+                        {"with_config", SELECT_IOC_WITH_CONFIG},
+                        {"total", SELECT_IOC_TOTAL}};
+                if (ioc_selection_raw.has_value())
+                {
+                    if (auto ioc_selection =
+                            ioc_selection_map.find(*ioc_selection_raw);
+                        ioc_selection != ioc_selection_map.end())
+                    {
+                        vfd_config.shared_cfg.ioc_selection =
+                            ioc_selection->second;
+                    }
+                    else
+                    {
+                        throw error::BackendConfigSchema(
+                            {"hdf5", "vfd", "ioc_selection"},
+                            "Unexpected value: '" + *ioc_selection_raw + "'.");
+                    }
+                }
+
+                // ... and set them
+                H5Pset_fapl_subfiling(m_fileAccessProperty, &vfd_config);
+            }
+            else
+            {
+                std::cerr << "[HDF5 Backend] The requested subfiling VFD of "
+                             "HDF5 requires the use of threaded MPI."
+                          << std::endl;
+            }
+#else
+            std::cerr
+                << "[HDF5 Backend] No support for the requested subfiling VFD "
+                   "found in the installed version of HDF5. Will continue with "
+                   "default settings. Tip: Configure a recent version of HDF5 "
+                   "with '-DHDF5_ENABLE_SUBFILING_VFD=ON'."
+                << std::endl;
+#endif
+        }
+        else
+        {
+            throw error::BackendConfigSchema(
+                {"hdf5", "vfd", "type"},
+                "Unknown value: '" + user_specified_type + "'.");
+        }
+
+        // unused params
+        auto shadow = m_config.invertShadow();
+        if (shadow.size() > 0)
+        {
+            switch (m_config.originallySpecifiedAs)
+            {
+            case json::SupportedLanguages::JSON:
+                std::cerr << "Warning: parts of the backend configuration for "
+                             "HDF5 remain unused:\n"
+                          << shadow << std::endl;
+                break;
+            case json::SupportedLanguages::TOML: {
+                auto asToml = json::jsonToToml(shadow);
+                std::cerr << "Warning: parts of the backend configuration for "
+                             "HDF5 remain unused:\n"
+                          << asToml << std::endl;
+                break;
+            }
+            }
+        }
+    }
 }
 
 ParallelHDF5IOHandlerImpl::~ParallelHDF5IOHandlerImpl()