Main implementation

Use magic number instead of API call (impl)

Author: franzpoeschel

include/openPMD/Dataset.hpp

@@ -22,7 +22,9 @@
 
 #include "openPMD/Datatype.hpp"
 
+#include <limits>
 #include <memory>
+#include <optional>
 #include <string>
 #include <type_traits>
 #include <vector>
@@ -37,6 +39,11 @@ class Dataset
     friend class RecordComponent;
 
 public:
+    enum : std::uint64_t
+    {
+        JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max()
+    };
+
     Dataset(Datatype, Extent, std::string options = "{}");
 
     /**
@@ -53,5 +60,9 @@ class Dataset
     Datatype dtype;
     uint8_t rank;
     std::string options = "{}"; //!< backend-dependent JSON configuration
+
+    bool empty() const;
+
+    std::optional<size_t> joinedDimension() const;
 };
 } // namespace openPMD

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -437,7 +437,7 @@ class ADIOS2IOHandlerImpl
     template <typename T>
     adios2::Variable<T> verifyDataset(
         Offset const &offset,
-        Extent const &extent,
+        Extent &extent,
         adios2::IO &IO,
         std::string const &var);
 }; // ADIOS2IOHandlerImpl

include/openPMD/IO/IOTask.hpp

@@ -349,6 +349,7 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
     Extent extent = {};
     Datatype dtype = Datatype::UNDEFINED;
     std::string options = "{}";
+    std::optional<size_t> joinedDimension;
 
     /** Warn about unused JSON paramters
      *

include/openPMD/RecordComponent.hpp

@@ -434,6 +434,11 @@ class RecordComponent : public BaseRecordComponent
     std::shared_ptr<internal::RecordComponentData> m_recordComponentData{
         new internal::RecordComponentData()};
 
+    template <typename T>
+    void verifyChunk(Offset const &, Extent const &) const;
+
+    void verifyChunk(Datatype, Offset const &, Extent const &) const;
+
     RecordComponent();
 
     // clang-format off

include/openPMD/RecordComponent.tpp

@@ -257,8 +257,17 @@ RecordComponent::storeChunk(T_ContiguousContainer &data, Offset o, Extent e)
     // default arguments
     //   offset = {0u}: expand to right dim {0u, 0u, ...}
     Offset offset = o;
-    if (o.size() == 1u && o.at(0) == 0u && dim > 1u)
-        offset = Offset(dim, 0u);
+    if (o.size() == 1u && o.at(0) == 0u)
+    {
+        if (joinedDimension().has_value())
+        {
+            offset.clear();
+        }
+        else if (dim > 1u)
+        {
+            offset = Offset(dim, 0u);
+        }
+    }
 
     //   extent = {-1u}: take full size
     Extent extent(dim, 1u);
@@ -276,38 +285,7 @@ template <typename T, typename F>
 inline DynamicMemoryView<T>
 RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
 {
-    if (constant())
-        throw std::runtime_error(
-            "Chunks cannot be written for a constant RecordComponent.");
-    if (empty())
-        throw std::runtime_error(
-            "Chunks cannot be written for an empty RecordComponent.");
-    Datatype dtype = determineDatatype<T>();
-    if (dtype != getDatatype())
-    {
-        std::ostringstream oss;
-        oss << "Datatypes of chunk data (" << dtype
-            << ") and record component (" << getDatatype() << ") do not match.";
-        throw std::runtime_error(oss.str());
-    }
-    uint8_t dim = getDimensionality();
-    if (e.size() != dim || o.size() != dim)
-    {
-        std::ostringstream oss;
-        oss << "Dimensionality of chunk ("
-            << "offset=" << o.size() << "D, "
-            << "extent=" << e.size() << "D) "
-            << "and record component (" << int(dim) << "D) "
-            << "do not match.";
-        throw std::runtime_error(oss.str());
-    }
-    Extent dse = getExtent();
-    for (uint8_t i = 0; i < dim; ++i)
-        if (dse[i] < o[i] + e[i])
-            throw std::runtime_error(
-                "Chunk does not reside inside dataset (Dimension on index " +
-                std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
-                " - Chunk: " + std::to_string(o[i] + e[i]) + ")");
+    verifyChunk<T>(o, e);
 
     /*
      * The openPMD backend might not yet know about this dataset.
@@ -332,6 +310,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
         dCreate.name = rc.m_name;
         dCreate.extent = getExtent();
         dCreate.dtype = getDatatype();
+        dCreate.joinedDimension = joinedDimension();
         if (!rc.m_dataset.has_value())
         {
             throw error::WrongAPIUsage(
@@ -372,4 +351,10 @@ RecordComponent::storeChunk(Offset offset, Extent extent)
 #endif
     });
 }
+
+template <typename T>
+void RecordComponent::verifyChunk(Offset const &o, Extent const &e) const
+{
+    verifyChunk(determineDatatype<T>(), o, e);
+}
 } // namespace openPMD

include/openPMD/backend/BaseRecordComponent.hpp

@@ -82,6 +82,8 @@ class BaseRecordComponent : public Attributable
      */
     bool constant() const;
 
+    std::optional<size_t> joinedDimension() const;
+
     /**
      * Get data chunks that are available to be loaded from the backend.
      * Note that this is backend-dependent information and the returned

include/openPMD/backend/PatchRecordComponent.hpp

@@ -95,6 +95,9 @@ class PatchRecordComponent : public BaseRecordComponent
     template <typename T>
     void store(uint64_t idx, T);
 
+    template <typename T>
+    void store(T);
+
     // clang-format off
 OPENPMD_private
     // clang-format on
@@ -222,4 +225,25 @@ inline void PatchRecordComponent::store(uint64_t idx, T data)
     auto &rc = get();
     rc.m_chunks.push(IOTask(this, std::move(dWrite)));
 }
+
+template <typename T>
+inline void PatchRecordComponent::store(T data)
+{
+    Datatype dtype = determineDatatype<T>();
+    if (dtype != getDatatype())
+    {
+        std::ostringstream oss;
+        oss << "Datatypes of patch data (" << dtype << ") and dataset ("
+            << getDatatype() << ") do not match.";
+        throw std::runtime_error(oss.str());
+    }
+
+    Parameter<Operation::WRITE_DATASET> dWrite;
+    dWrite.offset = {};
+    dWrite.extent = {1};
+    dWrite.dtype = dtype;
+    dWrite.data = std::make_shared<T>(data);
+    auto &rc = get();
+    rc.m_chunks.push(IOTask(this, std::move(dWrite)));
+}
 } // namespace openPMD

src/Dataset.cpp

@@ -19,6 +19,7 @@
  * If not, see <http://www.gnu.org/licenses/>.
  */
 #include "openPMD/Dataset.hpp"
+#include "openPMD/Error.hpp"
 
 #include <cstddef>
 #include <iostream>
@@ -30,7 +31,11 @@ Dataset::Dataset(Datatype d, Extent e, std::string options_in)
     , dtype{d}
     , rank{static_cast<uint8_t>(e.size())}
     , options{std::move(options_in)}
-{}
+{
+    // Call this in order to have early error message in case of wrong
+    // specification of joined dimensions
+    joinedDimension();
+}
 
 Dataset::Dataset(Extent e) : Dataset(Datatype::UNDEFINED, std::move(e))
 {}
@@ -49,4 +54,41 @@ Dataset &Dataset::extend(Extent newExtents)
     extent = newExtents;
     return *this;
 }
+
+bool Dataset::empty() const
+{
+    auto jd = joinedDimension();
+    for (size_t i = 0; i < extent.size(); ++i)
+    {
+        if (extent[i] == 0 && (!jd.has_value() || jd.value() != i))
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+std::optional<size_t> Dataset::joinedDimension() const
+{
+    std::optional<size_t> res;
+    for (size_t i = 0; i < extent.size(); ++i)
+    {
+        if (extent[i] == JOINED_DIMENSION)
+        {
+            if (res.has_value())
+            {
+                throw error::WrongAPIUsage(
+                    "Must specify JOINED_DIMENSION at most once (found at "
+                    "indices " +
+                    std::to_string(res.value()) + " and " + std::to_string(i) +
+                    ")");
+            }
+            else
+            {
+                res = i;
+            }
+        }
+    }
+    return res;
+}
 } // namespace openPMD

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -66,6 +66,21 @@ namespace openPMD
 
 #define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208)
 
+namespace
+{
+    std::optional<size_t> joinedDimension(adios2::Dims const &dims)
+    {
+        for (size_t i = 0; i < dims.size(); ++i)
+        {
+            if (dims[i] == adios2::JoinedDim)
+            {
+                return i;
+            }
+        }
+        return std::nullopt;
+    }
+} // namespace
+
 #if openPMD_HAVE_MPI
 
 ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl(
@@ -641,8 +656,11 @@ void ADIOS2IOHandlerImpl::createDataset(
                 varName + "' remain unused:\n");
 
         // cast from openPMD::Extent to adios2::Dims
-        adios2::Dims const shape(
-            parameters.extent.begin(), parameters.extent.end());
+        adios2::Dims shape(parameters.extent.begin(), parameters.extent.end());
+        if (auto jd = parameters.joinedDimension; jd.has_value())
+        {
+            shape[jd.value()] = adios2::JoinedDim;
+        }
 
         auto &fileData = getFileData(file, IfFileNotOpen::ThrowError);
         switchAdios2VariableType<detail::VariableDefiner>(
@@ -1518,7 +1536,7 @@ void ADIOS2IOHandlerImpl::dropFileData(InvalidatableFile file)
 template <typename T>
 adios2::Variable<T> ADIOS2IOHandlerImpl::verifyDataset(
     Offset const &offset,
-    Extent const &extent,
+    Extent &extent,
     adios2::IO &IO,
     std::string const &varName)
 {
@@ -1549,11 +1567,22 @@ adios2::Variable<T> ADIOS2IOHandlerImpl::verifyDataset(
                 std::to_string(requiredDim) + ", but has dimensionality " +
                 std::to_string(actualDim) + ")")
     }
+    auto joinedDim = joinedDimension(shape);
     for (unsigned int i = 0; i < actualDim; i++)
+    {
+        if (!joinedDim.has_value() || i != joinedDim.value())
+        {
+            VERIFY_ALWAYS(
+                offset[i] + extent[i] <= shape[i],
+                "[ADIOS2] Dataset access out of bounds.")
+        }
+    }
+
+    if (joinedDim.has_value())
     {
         VERIFY_ALWAYS(
-            offset[i] + extent[i] <= shape[i],
-            "[ADIOS2] Dataset access out of bounds.")
+            offset.empty(),
+            "[ADIOS2] Offset must be an empty vector in case of joined array.");
     }
 
     var.SetSelection(