Merge remote-tracking branch 'mainline/dev' into topic-pybind11-2.13-py-3.13

Author: ax3l

CMakeLists.txt

@@ -826,6 +826,7 @@ if(openPMD_BUILD_TESTING)
                 test/Files_SerialIO/close_and_reopen_test.cpp
                 test/Files_SerialIO/filebased_write_test.cpp
                 test/Files_SerialIO/issue_1744_unique_ptrs_at_close_time.cpp
+                test/Files_SerialIO/components_without_extent.cpp
             )
         elseif(${test_name} STREQUAL "ParallelIO" AND openPMD_HAVE_MPI)
             list(APPEND ${out_list}

docs/source/analysis/pandas.rst

@@ -50,6 +50,17 @@ One can also combine all iterations in a single dataframe like this:
    # like before but with a new column "iteration" and all particles
    print(df)
 
+Additionally, one can add additional openPMD particle species attributes, e.g.,
+from the `ED-PIC <https://github.com/openPMD/openPMD-standard/blob/1.1.0/EXT_ED-PIC.md#particle-records-macroparticles>`__ extension
+or `custom code properties <https://impactx.readthedocs.io/en/25.11/dataanalysis/dataanalysis.html#additional-beam-attributes>`__
+as extra dataframe columns:
+
+.. code-block:: python
+
+   df = s.to_df("electrons", attributes=["s_ref"])
+
+   # like before but with a new column "s_ref"
+   print(df)
 
 .. _analysis-pandas-ascii:
 

docs/source/details/backendconfig.rst

@@ -122,6 +122,11 @@ Or in a Series constructor JSON/TOML configuration:
     "hint_lazy_parsing_timeout": 20
   }
 
+As of openPMD-api 0.17.0, the parser verifies that all records within a mesh or within a particle species have consistent shapes / extents.
+This is used for filling in the shape for constant components that do not define it.
+In order to skip this check in the error case, the key ``{"verify_homogeneous_extents": false}`` may be set (alternatively ``export OPENPMD_VERIFY_HOMOGENEOUS_EXTENTS=0`` will do the same).
+This will help read datasets with inconsistent metadata definitions.
+
 The key ``resizable`` can be passed to ``Dataset`` options.
 It if set to ``{"resizable": true}``, this declares that it shall be allowed to increased the ``Extent`` of a ``Dataset`` via ``resetDataset()`` at a later time, i.e., after it has been first declared (and potentially written).
 For HDF5, resizable Datasets come with a performance penalty.

examples/11_particle_dataframe.py

@@ -38,26 +38,27 @@
     s = io.Series("../samples/git-sample/data%T.h5", io.Access.read_only)
     electrons = s.snapshots()[400].particles["electrons"]
 
-    # all particles
-    df = electrons.to_df()
+    # all particles, extra column for "particleShape" attribute
+    #                (from ED-PIC extension)
+    df = electrons.to_df(attributes=["particleShape"])
     print(type(df) is pd.DataFrame)
     print(df)
 
     # only first 100 particles
-    df = electrons.to_df(np.s_[:100])
+    df = electrons.to_df(slice=np.s_[:100])
     print(df)
 
     # all particles over all steps
-    df = s.to_df("electrons")
+    df = s.to_df("electrons", attributes=["particleShape"])
     print(df)
 
     if found_cudf:
         # all particles - to GPU
-        cdf = cudf.from_pandas(electrons.to_df())
+        cdf = cudf.from_pandas(electrons.to_df(attributes=["particleShape"]))
         print(cdf)
 
         # all particles over all steps - to GPU
-        cdf = s.to_cudf("electrons")
+        cdf = s.to_cudf("electrons", attributes=["particleShape"])
         print(cdf)
 
     # Particles
@@ -67,7 +68,7 @@
         # pickle capabilities, so we test this here:
         dask.config.set(scheduler='processes')
 
-        df = electrons.to_dask()
+        df = electrons.to_dask(attributes=["particleShape"])
         print(df)
 
         # check chunking of a variable

include/openPMD/Dataset.hpp

@@ -54,9 +54,21 @@ class Dataset
          */
         JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max(),
         /**
-         * Some backends (i.e. JSON and TOML in template mode) support the
-         * creation of dataset with undefined datatype and extent.
-         * The extent should be given as {UNDEFINED_EXTENT} for that.
+         * In some use cases, the extent needs not be specified.
+         * For these, specify Extent{UNDEFINED_EXTENT}.
+         * Use cases:
+         *
+         * 1. Some backends (i.e. JSON and TOML in template mode) support the
+         *    creation of datasets with undefined datatype and extent.
+         *    The extent should be given as {UNDEFINED_EXTENT} for that.
+         * 2. With openPMD 2.0, the shape of constant components may be omitted
+         *    in writing if it is defined somewhere else as part
+         *    of the same Mesh / Species.
+         *    (https://github.com/openPMD/openPMD-standard/pull/289)
+         *    When reading such datasets, the openPMD-api will try to fill in
+         *    the missing extents, so the extent for constistently-defined
+         *    datasets should ideally not be reported by the read-side API
+         *    as undefined.
          */
         UNDEFINED_EXTENT = std::numeric_limits<std::uint64_t>::max() - 1
     };
@@ -87,5 +99,8 @@ class Dataset
 
     std::optional<size_t> joinedDimension() const;
     static std::optional<size_t> joinedDimension(Extent const &);
+
+    bool undefinedExtent() const;
+    static bool undefinedExtent(Extent const &);
 };
 } // namespace openPMD

include/openPMD/IO/AbstractIOHandler.hpp

@@ -314,6 +314,7 @@ class AbstractIOHandler
     internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default;
     IterationEncoding m_encoding = IterationEncoding::groupBased;
     OpenpmdStandard m_standard = auxiliary::parseStandard(getStandardDefault());
+    bool m_verify_homogeneous_extents = true;
 }; // AbstractIOHandler
 
 } // namespace openPMD

include/openPMD/IO/IOTask.hpp

@@ -22,6 +22,7 @@
 
 #include "openPMD/ChunkInfo.hpp"
 #include "openPMD/Dataset.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/Streaming.hpp"
 #include "openPMD/auxiliary/Export.hpp"

include/openPMD/Record.hpp

@@ -53,7 +53,8 @@ class Record : public BaseRecord<RecordComponent>
 
     void
     flush_impl(std::string const &, internal::FlushParams const &) override;
-    void read();
+
+    [[nodiscard]] internal::HomogenizeExtents read();
 }; // Record
 
 template <typename T>

include/openPMD/RecordComponent.hpp

@@ -158,6 +158,8 @@ class RecordComponent : public BaseRecordComponent
      * * Shrinking any dimension's extent.
      * * Changing the number of dimensions.
      *
+     * The dataset extent may be empty to indicate undefined extents.
+     *
      * Backend support for resizing datasets:
      * * JSON: Supported
      * * ADIOS2: Supported as of ADIOS2 2.7.0
@@ -540,6 +542,26 @@ OPENPMD_protected
     void verifyChunk(Datatype, Offset const &, Extent const &) const;
 }; // RecordComponent
 
+namespace internal
+{
+    // Must put this after the definition of RecordComponent due to the
+    // deque<RecordComponent>
+    struct HomogenizeExtents
+    {
+        std::deque<RecordComponent> without_extent;
+        std::optional<Extent> retrieved_extent;
+        bool verify_homogeneous_extents = true;
+
+        explicit HomogenizeExtents();
+        HomogenizeExtents(bool verify_homogeneous_extents);
+
+        void check_extent(Attributable const &callsite, RecordComponent &);
+        auto merge(Attributable const &callsite, HomogenizeExtents)
+            -> HomogenizeExtents &;
+        void homogenize(Attributable const &callsite) &&;
+    };
+} // namespace internal
+
 } // namespace openPMD
 
 #include "openPMD/UndefDatatypeMacros.hpp"

include/openPMD/backend/Attributable.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/Error.hpp"
 #include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/ThrowError.hpp"
 #include "openPMD/auxiliary/OutOfRangeMsg.hpp"
@@ -29,6 +30,7 @@
 #include <cstddef>
 #include <map>
 #include <memory>
+#include <optional>
 #include <string>
 #include <type_traits>
 #include <vector>
@@ -54,6 +56,7 @@ namespace internal
 {
     class IterationData;
     class SeriesData;
+    struct HomogenizeExtents;
 
     class SharedAttributableData
     {
@@ -105,6 +108,7 @@ namespace internal
         friend class openPMD::Attributable;
 
         using SharedData_t = std::shared_ptr<SharedAttributableData>;
+        using A_MAP = SharedData_t::element_type::A_MAP;
 
     public:
         AttributableData();
@@ -155,6 +159,32 @@ namespace internal
                 std::shared_ptr<typename T::Data_t>(self, [](auto const *) {}));
             return res;
         }
+
+        inline auto attributes() -> A_MAP &
+        {
+            return operator*().m_attributes;
+        }
+        [[nodiscard]] inline auto attributes() const -> A_MAP const &
+        {
+            return operator*().m_attributes;
+        }
+        [[nodiscard]] inline auto readAttribute(std::string const &name) const
+            -> Attribute const &
+        {
+            auto const &attr = attributes();
+            if (auto it = attr.find(name); it != attr.end())
+            {
+                return it->second;
+            }
+            else
+            {
+                throw error::ReadError(
+                    error::AffectedObject::Attribute,
+                    error::Reason::NotFound,
+                    std::nullopt,
+                    "Not found: '" + name + "'.");
+            }
+        }
     };
 
     template <typename, typename>
@@ -213,6 +243,7 @@ class Attributable
     friend class StatefulSnapshotsContainer;
     friend class internal::AttributableData;
     friend class Snapshots;
+    friend struct internal::HomogenizeExtents;
 
 protected:
     // tag for internal constructor