Make shape an optional attribute for constant components (#1661)

* Make shape an optional attribute for constant components

* Adhere to the standard changes more closely

* Remove is_scalar distinction from flushing again

Both reasons for making the distinction now no longer valid

* Cleanup

* Homogenize extents

* Check for extent consistency, fill in undefined extents

* Make homogenization configurable

* Fix tests with this

* Add note on how to deactivate this verification

* WIP: Testing

* CI fixes

* Extend test

* Use JSON backend for this test

* Fix unfinished_iteration_test

* Bugfix: skip skipped compoments

* Use UNDEFINED_EXTENT for this instead of empty extent

* Document verify_homogeneous_extents option

* Fix typo

* Adapt test to stricter parsing

* Relax JSON schema

* uhmm.. remove merge conflict

Author: franzpoeschel

CMakeLists.txt

@@ -826,6 +826,7 @@ if(openPMD_BUILD_TESTING)
                 test/Files_SerialIO/close_and_reopen_test.cpp
                 test/Files_SerialIO/filebased_write_test.cpp
                 test/Files_SerialIO/issue_1744_unique_ptrs_at_close_time.cpp
+                test/Files_SerialIO/components_without_extent.cpp
             )
         elseif(${test_name} STREQUAL "ParallelIO" AND openPMD_HAVE_MPI)
             list(APPEND ${out_list}

docs/source/details/backendconfig.rst

@@ -122,6 +122,11 @@ Or in a Series constructor JSON/TOML configuration:
     "hint_lazy_parsing_timeout": 20
   }
 
+As of openPMD-api 0.17.0, the parser verifies that all records within a mesh or within a particle species have consistent shapes / extents.
+This is used for filling in the shape for constant components that do not define it.
+In order to skip this check in the error case, the key ``{"verify_homogeneous_extents": false}`` may be set (alternatively ``export OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS=0`` will do the same).
+This will help read datasets with inconsistent metadata definitions.
+
 The key ``resizable`` can be passed to ``Dataset`` options.
 It if set to ``{"resizable": true}``, this declares that it shall be allowed to increased the ``Extent`` of a ``Dataset`` via ``resetDataset()`` at a later time, i.e., after it has been first declared (and potentially written).
 For HDF5, resizable Datasets come with a performance penalty.

include/openPMD/Dataset.hpp

@@ -54,9 +54,21 @@ class Dataset
          */
         JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max(),
         /**
-         * Some backends (i.e. JSON and TOML in template mode) support the
-         * creation of dataset with undefined datatype and extent.
-         * The extent should be given as {UNDEFINED_EXTENT} for that.
+         * In some use cases, the extent needs not be specified.
+         * For these, specify Extent{UNDEFINED_EXTENT}.
+         * Use cases:
+         *
+         * 1. Some backends (i.e. JSON and TOML in template mode) support the
+         *    creation of datasets with undefined datatype and extent.
+         *    The extent should be given as {UNDEFINED_EXTENT} for that.
+         * 2. With openPMD 2.0, the shape of constant components may be omitted
+         *    in writing if it is defined somewhere else as part
+         *    of the same Mesh / Species.
+         *    (https://github.com/openPMD/openPMD-standard/pull/289)
+         *    When reading such datasets, the openPMD-api will try to fill in
+         *    the missing extents, so the extent for constistently-defined
+         *    datasets should ideally not be reported by the read-side API
+         *    as undefined.
          */
         UNDEFINED_EXTENT = std::numeric_limits<std::uint64_t>::max() - 1
     };
@@ -87,5 +99,8 @@ class Dataset
 
     std::optional<size_t> joinedDimension() const;
     static std::optional<size_t> joinedDimension(Extent const &);
+
+    bool undefinedExtent() const;
+    static bool undefinedExtent(Extent const &);
 };
 } // namespace openPMD

include/openPMD/IO/AbstractIOHandler.hpp

@@ -314,6 +314,7 @@ class AbstractIOHandler
     internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default;
     IterationEncoding m_encoding = IterationEncoding::groupBased;
     OpenpmdStandard m_standard = auxiliary::parseStandard(getStandardDefault());
+    bool m_verify_homogeneous_extents = true;
 }; // AbstractIOHandler
 
 } // namespace openPMD

include/openPMD/IO/IOTask.hpp

@@ -22,6 +22,7 @@
 
 #include "openPMD/ChunkInfo.hpp"
 #include "openPMD/Dataset.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/Streaming.hpp"
 #include "openPMD/auxiliary/Export.hpp"

include/openPMD/Record.hpp

@@ -53,7 +53,8 @@ class Record : public BaseRecord<RecordComponent>
 
     void
     flush_impl(std::string const &, internal::FlushParams const &) override;
-    void read();
+
+    [[nodiscard]] internal::HomogenizeExtents read();
 }; // Record
 
 template <typename T>

include/openPMD/RecordComponent.hpp

@@ -158,6 +158,8 @@ class RecordComponent : public BaseRecordComponent
      * * Shrinking any dimension's extent.
      * * Changing the number of dimensions.
      *
+     * The dataset extent may be empty to indicate undefined extents.
+     *
      * Backend support for resizing datasets:
      * * JSON: Supported
      * * ADIOS2: Supported as of ADIOS2 2.7.0
@@ -540,6 +542,26 @@ OPENPMD_protected
     void verifyChunk(Datatype, Offset const &, Extent const &) const;
 }; // RecordComponent
 
+namespace internal
+{
+    // Must put this after the definition of RecordComponent due to the
+    // deque<RecordComponent>
+    struct HomogenizeExtents
+    {
+        std::deque<RecordComponent> without_extent;
+        std::optional<Extent> retrieved_extent;
+        bool verify_homogeneous_extents = true;
+
+        explicit HomogenizeExtents();
+        HomogenizeExtents(bool verify_homogeneous_extents);
+
+        void check_extent(Attributable const &callsite, RecordComponent &);
+        auto merge(Attributable const &callsite, HomogenizeExtents)
+            -> HomogenizeExtents &;
+        void homogenize(Attributable const &callsite) &&;
+    };
+} // namespace internal
+
 } // namespace openPMD
 
 #include "openPMD/UndefDatatypeMacros.hpp"

include/openPMD/backend/Attributable.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/Error.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/ThrowError.hpp"
 #include "openPMD/auxiliary/OutOfRangeMsg.hpp"
@@ -29,6 +30,7 @@
 #include <cstddef>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
 #include <type_traits>
 #include <vector>
@@ -54,6 +56,7 @@ namespace internal
 {
     class IterationData;
     class SeriesData;
+    struct HomogenizeExtents;
 
     class SharedAttributableData
     {
@@ -105,6 +108,7 @@ namespace internal
         friend class openPMD::Attributable;
 
         using SharedData_t = std::shared_ptr<SharedAttributableData>;
+        using A_MAP = SharedData_t::element_type::A_MAP;
 
     public:
         AttributableData();
@@ -155,6 +159,32 @@ namespace internal
                 std::shared_ptr<typename T::Data_t>(self, [](auto const *) {}));
             return res;
         }
+
+        inline auto attributes() -> A_MAP &
+        {
+            return operator*().m_attributes;
+        }
+        [[nodiscard]] inline auto attributes() const -> A_MAP const &
+        {
+            return operator*().m_attributes;
+        }
+        [[nodiscard]] inline auto readAttribute(std::string const &name) const
+            -> Attribute const &
+        {
+            auto const &attr = attributes();
+            if (auto it = attr.find(name); it != attr.end())
+            {
+                return it->second;
+            }
+            else
+            {
+                throw error::ReadError(
+                    error::AffectedObject::Attribute,
+                    error::Reason::NotFound,
+                    std::nullopt,
+                    "Not found: '" + name + "'.");
+            }
+        }
     };
 
     template <typename, typename>
@@ -213,6 +243,7 @@ class Attributable
     friend class StatefulSnapshotsContainer;
     friend class internal::AttributableData;
     friend class Snapshots;
+    friend struct internal::HomogenizeExtents;
 
 protected:
     // tag for internal constructor

share/openPMD/json_schema/record_component.toml

@@ -44,7 +44,10 @@ title = "Either array or constant"
 
 [allOf.if]
 required = ["attributes"]
-properties.attributes.required = ["shape", "value"]
+# shape optional as of https://github.com/openPMD/openPMD-api/pull/1661
+# JSON schema not powerful enough to check that the shape must be defined
+# at least once (just like we do not check that the shape is consistent).
+properties.attributes.required = ["value"]
 
 [allOf.then]
 title = "Constant dataset"

src/Dataset.cpp

@@ -96,4 +96,13 @@ std::optional<size_t> Dataset::joinedDimension(Extent const &extent)
     }
     return res;
 }
+
+bool Dataset::undefinedExtent() const
+{
+    return undefinedExtent(extent);
+}
+bool Dataset::undefinedExtent(Extent const &e)
+{
+    return e.size() == 1 && e.at(0) == Dataset::UNDEFINED_EXTENT;
+}
 } // namespace openPMD