Skip to content

Commit 2eeea7c

Browse files
franzpoeschelfranzpoeschel
committed
Fundamentally functional version
1 parent f740cb4 commit 2eeea7c

File tree

13 files changed

+250
-104
lines changed

13 files changed

+250
-104
lines changed

‎include/openPMD/Dataset.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "openPMD/Datatype.hpp"
2424

2525
#include <memory>
26+
#include <optional>
2627
#include <string>
2728
#include <type_traits>
2829
#include <vector>
@@ -52,6 +53,7 @@ class Dataset
5253
Extent extent;
5354
Datatype dtype;
5455
uint8_t rank;
56+
std::optional<size_t> joinedDimension;
5557
std::string options = "{}"; //!< backend-dependent JSON configuration
5658
};
5759
} // namespace openPMD

‎include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -437,7 +437,7 @@ class ADIOS2IOHandlerImpl
437437
template <typename T>
438438
adios2::Variable<T> verifyDataset(
439439
Offset const &offset,
440-
Extent const &extent,
440+
Extent &extent,
441441
adios2::IO &IO,
442442
std::string const &var);
443443
}; // ADIOS2IOHandlerImpl

‎include/openPMD/IO/IOTask.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
349349
Extent extent = {};
350350
Datatype dtype = Datatype::UNDEFINED;
351351
std::string options = "{}";
352+
std::optional<size_t> joinedDimension;
352353

353354
/** Warn about unused JSON paramters
354355
*

‎include/openPMD/RecordComponent.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,11 @@ class RecordComponent : public BaseRecordComponent
435435
std::shared_ptr<internal::RecordComponentData> m_recordComponentData{
436436
new internal::RecordComponentData()};
437437

438+
template <typename T>
439+
void verifyChunk(Offset const &, Extent const &) const;
440+
441+
void verifyChunk(Datatype, Offset const &, Extent const &) const;
442+
438443
RecordComponent();
439444

440445
// clang-format off

‎include/openPMD/RecordComponent.tpp‎

Lines changed: 19 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,17 @@ RecordComponent::storeChunk(T_ContiguousContainer &data, Offset o, Extent e)
262262
// default arguments
263263
// offset = {0u}: expand to right dim {0u, 0u, ...}
264264
Offset offset = o;
265-
if( o.size() == 1u && o.at(0) == 0u && dim > 1u )
266-
offset = Offset(dim, 0u);
265+
if (o.size() == 1u && o.at(0) == 0u)
266+
{
267+
if (joinedDimension().has_value())
268+
{
269+
offset.clear();
270+
}
271+
else if (dim > 1u)
272+
{
273+
offset = Offset(dim, 0u);
274+
}
275+
}
267276

268277
// extent = {-1u}: take full size
269278
Extent extent(dim, 1u);
@@ -284,42 +293,7 @@ template< typename T, typename F >
284293
inline DynamicMemoryView< T >
285294
RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer )
286295
{
287-
if( constant() )
288-
throw std::runtime_error(
289-
"Chunks cannot be written for a constant RecordComponent." );
290-
if( empty() )
291-
throw std::runtime_error(
292-
"Chunks cannot be written for an empty RecordComponent." );
293-
Datatype dtype = determineDatatype<T>();
294-
if( dtype != getDatatype() )
295-
{
296-
std::ostringstream oss;
297-
oss << "Datatypes of chunk data ("
298-
<< dtype
299-
<< ") and record component ("
300-
<< getDatatype()
301-
<< ") do not match.";
302-
throw std::runtime_error(oss.str());
303-
}
304-
uint8_t dim = getDimensionality();
305-
if( e.size() != dim || o.size() != dim )
306-
{
307-
std::ostringstream oss;
308-
oss << "Dimensionality of chunk ("
309-
<< "offset=" << o.size() << "D, "
310-
<< "extent=" << e.size() << "D) "
311-
<< "and record component ("
312-
<< int(dim) << "D) "
313-
<< "do not match.";
314-
throw std::runtime_error(oss.str());
315-
}
316-
Extent dse = getExtent();
317-
for( uint8_t i = 0; i < dim; ++i )
318-
if( dse[i] < o[i] + e[i] )
319-
throw std::runtime_error("Chunk does not reside inside dataset (Dimension on index " + std::to_string(i)
320-
+ ". DS: " + std::to_string(dse[i])
321-
+ " - Chunk: " + std::to_string(o[i] + e[i])
322-
+ ")");
296+
verifyChunk<T>(o, e);
323297

324298
/*
325299
* The openPMD backend might not yet know about this dataset.
@@ -344,6 +318,7 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer )
344318
dCreate.name = rc.m_name;
345319
dCreate.extent = getExtent();
346320
dCreate.dtype = getDatatype();
321+
dCreate.joinedDimension = joinedDimension();
347322
if (!rc.m_dataset.has_value())
348323
{
349324
throw error::WrongAPIUsage(
@@ -388,4 +363,10 @@ RecordComponent::storeChunk( Offset offset, Extent extent )
388363
#endif
389364
} );
390365
}
366+
367+
template <typename T>
368+
void RecordComponent::verifyChunk(Offset const &o, Extent const &e) const
369+
{
370+
verifyChunk(determineDatatype<T>(), o, e);
391371
}
372+
} // namespace openPMD

‎include/openPMD/backend/BaseRecordComponent.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ class BaseRecordComponent : public Attributable
8282
*/
8383
bool constant() const;
8484

85+
std::optional<size_t> joinedDimension() const;
86+
8587
/**
8688
* Get data chunks that are available to be loaded from the backend.
8789
* Note that this is backend-dependent information and the returned

‎include/openPMD/backend/PatchRecordComponent.hpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ class PatchRecordComponent : public BaseRecordComponent
9494
template <typename T>
9595
void store(uint64_t idx, T);
9696

97+
template <typename T>
98+
void store(T);
99+
97100
// clang-format off
98101
OPENPMD_private
99102
// clang-format on
@@ -220,4 +223,25 @@ inline void PatchRecordComponent::store(uint64_t idx, T data)
220223
auto &rc = get();
221224
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
222225
}
226+
227+
template <typename T>
228+
inline void PatchRecordComponent::store(T data)
229+
{
230+
Datatype dtype = determineDatatype<T>();
231+
if (dtype != getDatatype())
232+
{
233+
std::ostringstream oss;
234+
oss << "Datatypes of patch data (" << dtype << ") and dataset ("
235+
<< getDatatype() << ") do not match.";
236+
throw std::runtime_error(oss.str());
237+
}
238+
239+
Parameter<Operation::WRITE_DATASET> dWrite;
240+
dWrite.offset = {};
241+
dWrite.extent = {1};
242+
dWrite.dtype = dtype;
243+
dWrite.data = std::make_shared<T>(data);
244+
auto &rc = get();
245+
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
246+
}
223247
} // namespace openPMD

‎src/IO/ADIOS/ADIOS2IOHandler.cpp‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,21 @@ namespace openPMD
6666

6767
#define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208)
6868

69+
namespace
70+
{
71+
std::optional<size_t> joinedDimension(adios2::Dims const &dims)
72+
{
73+
for (size_t i = 0; i < dims.size(); ++i)
74+
{
75+
if (dims[i] == adios2::JoinedDim)
76+
{
77+
return i;
78+
}
79+
}
80+
return std::nullopt;
81+
}
82+
} // namespace
83+
6984
#if openPMD_HAVE_MPI
7085

7186
ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl(
@@ -641,8 +656,11 @@ void ADIOS2IOHandlerImpl::createDataset(
641656
varName + "' remain unused:\n");
642657

643658
// cast from openPMD::Extent to adios2::Dims
644-
adios2::Dims const shape(
645-
parameters.extent.begin(), parameters.extent.end());
659+
adios2::Dims shape(parameters.extent.begin(), parameters.extent.end());
660+
if (auto jd = parameters.joinedDimension; jd.has_value())
661+
{
662+
shape[jd.value()] = adios2::JoinedDim;
663+
}
646664

647665
auto &fileData = getFileData(file, IfFileNotOpen::ThrowError);
648666
switchAdios2VariableType<detail::VariableDefiner>(
@@ -1518,7 +1536,7 @@ void ADIOS2IOHandlerImpl::dropFileData(InvalidatableFile file)
15181536
template <typename T>
15191537
adios2::Variable<T> ADIOS2IOHandlerImpl::verifyDataset(
15201538
Offset const &offset,
1521-
Extent const &extent,
1539+
Extent &extent,
15221540
adios2::IO &IO,
15231541
std::string const &varName)
15241542
{
@@ -1549,11 +1567,22 @@ adios2::Variable<T> ADIOS2IOHandlerImpl::verifyDataset(
15491567
std::to_string(requiredDim) + ", but has dimensionality " +
15501568
std::to_string(actualDim) + ")")
15511569
}
1570+
auto joinedDim = joinedDimension(shape);
15521571
for (unsigned int i = 0; i < actualDim; i++)
1572+
{
1573+
if (!joinedDim.has_value() || i != joinedDim.value())
1574+
{
1575+
VERIFY_ALWAYS(
1576+
offset[i] + extent[i] <= shape[i],
1577+
"[ADIOS2] Dataset access out of bounds.")
1578+
}
1579+
}
1580+
1581+
if (joinedDim.has_value())
15531582
{
15541583
VERIFY_ALWAYS(
1555-
offset[i] + extent[i] <= shape[i],
1556-
"[ADIOS2] Dataset access out of bounds.")
1584+
offset.empty(),
1585+
"[ADIOS2] Offset must be an empty vector in case of joined array.");
15571586
}
15581587

15591588
var.SetSelection(

‎src/RecordComponent.cpp‎

Lines changed: 67 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ void RecordComponent::flush(
271271
dCreate.extent = getExtent();
272272
dCreate.dtype = getDatatype();
273273
dCreate.options = rc.m_dataset.value().options;
274+
dCreate.joinedDimension = joinedDimension();
274275
IOHandler()->enqueue(IOTask(this, dCreate));
275276
}
276277
}
@@ -407,6 +408,21 @@ bool RecordComponent::dirtyRecursive() const
407408

408409
void RecordComponent::storeChunk(
409410
auxiliary::WriteBuffer buffer, Datatype dtype, Offset o, Extent e)
411+
{
412+
verifyChunk(dtype, o, e);
413+
414+
Parameter<Operation::WRITE_DATASET> dWrite;
415+
dWrite.offset = o;
416+
dWrite.extent = e;
417+
dWrite.dtype = dtype;
418+
/* std::static_pointer_cast correctly reference-counts the pointer */
419+
dWrite.data = std::move(buffer);
420+
auto &rc = get();
421+
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
422+
}
423+
424+
void RecordComponent::verifyChunk(
425+
Datatype dtype, Offset const &o, Extent const &e) const
410426
{
411427
if (constant())
412428
throw std::runtime_error(
@@ -422,31 +438,58 @@ void RecordComponent::storeChunk(
422438
throw std::runtime_error(oss.str());
423439
}
424440
uint8_t dim = getDimensionality();
425-
if (e.size() != dim || o.size() != dim)
426-
{
427-
std::ostringstream oss;
428-
oss << "Dimensionality of chunk ("
429-
<< "offset=" << o.size() << "D, "
430-
<< "extent=" << e.size() << "D) "
431-
<< "and record component (" << int(dim) << "D) "
432-
<< "do not match.";
433-
throw std::runtime_error(oss.str());
434-
}
435441
Extent dse = getExtent();
436-
for (uint8_t i = 0; i < dim; ++i)
437-
if (dse[i] < o[i] + e[i])
438-
throw std::runtime_error(
439-
"Chunk does not reside inside dataset (Dimension on index " +
440-
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
441-
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
442442

443-
Parameter<Operation::WRITE_DATASET> dWrite;
444-
dWrite.offset = o;
445-
dWrite.extent = e;
446-
dWrite.dtype = dtype;
447-
/* std::static_pointer_cast correctly reference-counts the pointer */
448-
dWrite.data = std::move(buffer);
449-
auto &rc = get();
450-
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
443+
if (auto jd = joinedDimension(); jd.has_value())
444+
{
445+
if (o.size() != 0)
446+
{
447+
std::ostringstream oss;
448+
oss << "Joined array: Must specify an empty offset (given: "
449+
<< "offset=" << o.size() << "D, "
450+
<< "extent=" << e.size() << "D).";
451+
throw std::runtime_error(oss.str());
452+
}
453+
if (e.size() != dim)
454+
{
455+
std::ostringstream oss;
456+
oss << "Joined array: Dimensionalities of chunk extent and dataset "
457+
"extent must be equivalent (given: "
458+
<< "offset=" << o.size() << "D, "
459+
<< "extent=" << e.size() << "D).";
460+
throw std::runtime_error(oss.str());
461+
}
462+
for (size_t i = 0; i < dim; ++i)
463+
{
464+
if (i != jd.value() && e[i] != dse[i])
465+
{
466+
throw std::runtime_error(
467+
"Joined array: Chunk extent on non-joined dimensions must "
468+
"be equivalent to dataset extents (Dimension on index " +
469+
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
470+
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
471+
}
472+
}
473+
}
474+
else
475+
{
476+
if (e.size() != dim || o.size() != dim)
477+
{
478+
std::ostringstream oss;
479+
oss << "Dimensionality of chunk ("
480+
<< "offset=" << o.size() << "D, "
481+
<< "extent=" << e.size() << "D) "
482+
<< "and record component (" << int(dim) << "D) "
483+
<< "do not match.";
484+
throw std::runtime_error(oss.str());
485+
}
486+
for (uint8_t i = 0; i < dim; ++i)
487+
if (dse[i] < o[i] + e[i])
488+
throw std::runtime_error(
489+
"Chunk does not reside inside dataset (Dimension on "
490+
"index " +
491+
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
492+
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
493+
}
451494
}
452495
} // namespace openPMD

‎src/backend/BaseRecordComponent.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,19 @@ bool BaseRecordComponent::constant() const
6565
return get().m_isConstant;
6666
}
6767

68+
std::optional<size_t> BaseRecordComponent::joinedDimension() const
69+
{
70+
auto &rc = get();
71+
if (rc.m_dataset.has_value())
72+
{
73+
return rc.m_dataset.value().joinedDimension;
74+
}
75+
else
76+
{
77+
return false;
78+
}
79+
}
80+
6881
ChunkTable BaseRecordComponent::availableChunks()
6982
{
7083
auto &rc = get();

0 commit comments

Comments
 (0)