Skip to content

Commit 0560f2e

Browse files
franzpoeschelfranzpoeschel
committed
Fundamentally functional version
1 parent 3acf565 commit 0560f2e

File tree

13 files changed

+250
-100
lines changed

13 files changed

+250
-100
lines changed

‎include/openPMD/Dataset.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "openPMD/Datatype.hpp"
2424

2525
#include <memory>
26+
#include <optional>
2627
#include <string>
2728
#include <type_traits>
2829
#include <vector>
@@ -52,6 +53,7 @@ class Dataset
5253
Extent extent;
5354
Datatype dtype;
5455
uint8_t rank;
56+
std::optional<size_t> joinedDimension;
5557
std::string options = "{}"; //!< backend-dependent JSON configuration
5658
};
5759
} // namespace openPMD

‎include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -437,7 +437,7 @@ class ADIOS2IOHandlerImpl
437437
template <typename T>
438438
adios2::Variable<T> verifyDataset(
439439
Offset const &offset,
440-
Extent const &extent,
440+
Extent &extent,
441441
adios2::IO &IO,
442442
std::string const &var);
443443
}; // ADIOS2IOHandlerImpl

‎include/openPMD/IO/IOTask.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
349349
Extent extent = {};
350350
Datatype dtype = Datatype::UNDEFINED;
351351
std::string options = "{}";
352+
std::optional<size_t> joinedDimension;
352353

353354
/** Warn about unused JSON paramters
354355
*

‎include/openPMD/RecordComponent.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,11 @@ class RecordComponent : public BaseRecordComponent
434434
std::shared_ptr<internal::RecordComponentData> m_recordComponentData{
435435
new internal::RecordComponentData()};
436436

437+
template <typename T>
438+
void verifyChunk(Offset const &, Extent const &) const;
439+
440+
void verifyChunk(Datatype, Offset const &, Extent const &) const;
441+
437442
RecordComponent();
438443

439444
// clang-format off

‎include/openPMD/RecordComponent.tpp‎

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -257,8 +257,17 @@ RecordComponent::storeChunk(T_ContiguousContainer &data, Offset o, Extent e)
257257
// default arguments
258258
// offset = {0u}: expand to right dim {0u, 0u, ...}
259259
Offset offset = o;
260-
if (o.size() == 1u && o.at(0) == 0u && dim > 1u)
261-
offset = Offset(dim, 0u);
260+
if (o.size() == 1u && o.at(0) == 0u)
261+
{
262+
if (joinedDimension().has_value())
263+
{
264+
offset.clear();
265+
}
266+
else if (dim > 1u)
267+
{
268+
offset = Offset(dim, 0u);
269+
}
270+
}
262271

263272
// extent = {-1u}: take full size
264273
Extent extent(dim, 1u);
@@ -276,38 +285,7 @@ template <typename T, typename F>
276285
inline DynamicMemoryView<T>
277286
RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
278287
{
279-
if (constant())
280-
throw std::runtime_error(
281-
"Chunks cannot be written for a constant RecordComponent.");
282-
if (empty())
283-
throw std::runtime_error(
284-
"Chunks cannot be written for an empty RecordComponent.");
285-
Datatype dtype = determineDatatype<T>();
286-
if (dtype != getDatatype())
287-
{
288-
std::ostringstream oss;
289-
oss << "Datatypes of chunk data (" << dtype
290-
<< ") and record component (" << getDatatype() << ") do not match.";
291-
throw std::runtime_error(oss.str());
292-
}
293-
uint8_t dim = getDimensionality();
294-
if (e.size() != dim || o.size() != dim)
295-
{
296-
std::ostringstream oss;
297-
oss << "Dimensionality of chunk ("
298-
<< "offset=" << o.size() << "D, "
299-
<< "extent=" << e.size() << "D) "
300-
<< "and record component (" << int(dim) << "D) "
301-
<< "do not match.";
302-
throw std::runtime_error(oss.str());
303-
}
304-
Extent dse = getExtent();
305-
for (uint8_t i = 0; i < dim; ++i)
306-
if (dse[i] < o[i] + e[i])
307-
throw std::runtime_error(
308-
"Chunk does not reside inside dataset (Dimension on index " +
309-
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
310-
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
288+
verifyChunk<T>(o, e);
311289

312290
/*
313291
* The openPMD backend might not yet know about this dataset.
@@ -332,6 +310,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
332310
dCreate.name = rc.m_name;
333311
dCreate.extent = getExtent();
334312
dCreate.dtype = getDatatype();
313+
dCreate.joinedDimension = joinedDimension();
335314
if (!rc.m_dataset.has_value())
336315
{
337316
throw error::WrongAPIUsage(
@@ -372,4 +351,10 @@ RecordComponent::storeChunk(Offset offset, Extent extent)
372351
#endif
373352
});
374353
}
354+
355+
template <typename T>
356+
void RecordComponent::verifyChunk(Offset const &o, Extent const &e) const
357+
{
358+
verifyChunk(determineDatatype<T>(), o, e);
359+
}
375360
} // namespace openPMD

‎include/openPMD/backend/BaseRecordComponent.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ class BaseRecordComponent : public Attributable
8282
*/
8383
bool constant() const;
8484

85+
std::optional<size_t> joinedDimension() const;
86+
8587
/**
8688
* Get data chunks that are available to be loaded from the backend.
8789
* Note that this is backend-dependent information and the returned

‎include/openPMD/backend/PatchRecordComponent.hpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ class PatchRecordComponent : public BaseRecordComponent
9595
template <typename T>
9696
void store(uint64_t idx, T);
9797

98+
template <typename T>
99+
void store(T);
100+
98101
// clang-format off
99102
OPENPMD_private
100103
// clang-format on
@@ -222,4 +225,25 @@ inline void PatchRecordComponent::store(uint64_t idx, T data)
222225
auto &rc = get();
223226
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
224227
}
228+
229+
template <typename T>
230+
inline void PatchRecordComponent::store(T data)
231+
{
232+
Datatype dtype = determineDatatype<T>();
233+
if (dtype != getDatatype())
234+
{
235+
std::ostringstream oss;
236+
oss << "Datatypes of patch data (" << dtype << ") and dataset ("
237+
<< getDatatype() << ") do not match.";
238+
throw std::runtime_error(oss.str());
239+
}
240+
241+
Parameter<Operation::WRITE_DATASET> dWrite;
242+
dWrite.offset = {};
243+
dWrite.extent = {1};
244+
dWrite.dtype = dtype;
245+
dWrite.data = std::make_shared<T>(data);
246+
auto &rc = get();
247+
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
248+
}
225249
} // namespace openPMD

‎src/IO/ADIOS/ADIOS2IOHandler.cpp‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,21 @@ namespace openPMD
6666

6767
#define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208)
6868

69+
namespace
70+
{
71+
std::optional<size_t> joinedDimension(adios2::Dims const &dims)
72+
{
73+
for (size_t i = 0; i < dims.size(); ++i)
74+
{
75+
if (dims[i] == adios2::JoinedDim)
76+
{
77+
return i;
78+
}
79+
}
80+
return std::nullopt;
81+
}
82+
} // namespace
83+
6984
#if openPMD_HAVE_MPI
7085

7186
ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl(
@@ -641,8 +656,11 @@ void ADIOS2IOHandlerImpl::createDataset(
641656
varName + "' remain unused:\n");
642657

643658
// cast from openPMD::Extent to adios2::Dims
644-
adios2::Dims const shape(
645-
parameters.extent.begin(), parameters.extent.end());
659+
adios2::Dims shape(parameters.extent.begin(), parameters.extent.end());
660+
if (auto jd = parameters.joinedDimension; jd.has_value())
661+
{
662+
shape[jd.value()] = adios2::JoinedDim;
663+
}
646664

647665
auto &fileData = getFileData(file, IfFileNotOpen::ThrowError);
648666
switchAdios2VariableType<detail::VariableDefiner>(
@@ -1518,7 +1536,7 @@ void ADIOS2IOHandlerImpl::dropFileData(InvalidatableFile file)
15181536
template <typename T>
15191537
adios2::Variable<T> ADIOS2IOHandlerImpl::verifyDataset(
15201538
Offset const &offset,
1521-
Extent const &extent,
1539+
Extent &extent,
15221540
adios2::IO &IO,
15231541
std::string const &varName)
15241542
{
@@ -1549,11 +1567,22 @@ adios2::Variable<T> ADIOS2IOHandlerImpl::verifyDataset(
15491567
std::to_string(requiredDim) + ", but has dimensionality " +
15501568
std::to_string(actualDim) + ")")
15511569
}
1570+
auto joinedDim = joinedDimension(shape);
15521571
for (unsigned int i = 0; i < actualDim; i++)
1572+
{
1573+
if (!joinedDim.has_value() || i != joinedDim.value())
1574+
{
1575+
VERIFY_ALWAYS(
1576+
offset[i] + extent[i] <= shape[i],
1577+
"[ADIOS2] Dataset access out of bounds.")
1578+
}
1579+
}
1580+
1581+
if (joinedDim.has_value())
15531582
{
15541583
VERIFY_ALWAYS(
1555-
offset[i] + extent[i] <= shape[i],
1556-
"[ADIOS2] Dataset access out of bounds.")
1584+
offset.empty(),
1585+
"[ADIOS2] Offset must be an empty vector in case of joined array.");
15571586
}
15581587

15591588
var.SetSelection(

‎src/RecordComponent.cpp‎

Lines changed: 67 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,7 @@ void RecordComponent::flush(
283283
dCreate.extent = getExtent();
284284
dCreate.dtype = getDatatype();
285285
dCreate.options = rc.m_dataset.value().options;
286+
dCreate.joinedDimension = joinedDimension();
286287
IOHandler()->enqueue(IOTask(this, dCreate));
287288
}
288289
}
@@ -419,6 +420,21 @@ bool RecordComponent::dirtyRecursive() const
419420

420421
void RecordComponent::storeChunk(
421422
auxiliary::WriteBuffer buffer, Datatype dtype, Offset o, Extent e)
423+
{
424+
verifyChunk(dtype, o, e);
425+
426+
Parameter<Operation::WRITE_DATASET> dWrite;
427+
dWrite.offset = o;
428+
dWrite.extent = e;
429+
dWrite.dtype = dtype;
430+
/* std::static_pointer_cast correctly reference-counts the pointer */
431+
dWrite.data = std::move(buffer);
432+
auto &rc = get();
433+
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
434+
}
435+
436+
void RecordComponent::verifyChunk(
437+
Datatype dtype, Offset const &o, Extent const &e) const
422438
{
423439
if (constant())
424440
throw std::runtime_error(
@@ -434,31 +450,58 @@ void RecordComponent::storeChunk(
434450
throw std::runtime_error(oss.str());
435451
}
436452
uint8_t dim = getDimensionality();
437-
if (e.size() != dim || o.size() != dim)
438-
{
439-
std::ostringstream oss;
440-
oss << "Dimensionality of chunk ("
441-
<< "offset=" << o.size() << "D, "
442-
<< "extent=" << e.size() << "D) "
443-
<< "and record component (" << int(dim) << "D) "
444-
<< "do not match.";
445-
throw std::runtime_error(oss.str());
446-
}
447453
Extent dse = getExtent();
448-
for (uint8_t i = 0; i < dim; ++i)
449-
if (dse[i] < o[i] + e[i])
450-
throw std::runtime_error(
451-
"Chunk does not reside inside dataset (Dimension on index " +
452-
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
453-
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
454454

455-
Parameter<Operation::WRITE_DATASET> dWrite;
456-
dWrite.offset = o;
457-
dWrite.extent = e;
458-
dWrite.dtype = dtype;
459-
/* std::static_pointer_cast correctly reference-counts the pointer */
460-
dWrite.data = std::move(buffer);
461-
auto &rc = get();
462-
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
455+
if (auto jd = joinedDimension(); jd.has_value())
456+
{
457+
if (o.size() != 0)
458+
{
459+
std::ostringstream oss;
460+
oss << "Joined array: Must specify an empty offset (given: "
461+
<< "offset=" << o.size() << "D, "
462+
<< "extent=" << e.size() << "D).";
463+
throw std::runtime_error(oss.str());
464+
}
465+
if (e.size() != dim)
466+
{
467+
std::ostringstream oss;
468+
oss << "Joined array: Dimensionalities of chunk extent and dataset "
469+
"extent must be equivalent (given: "
470+
<< "offset=" << o.size() << "D, "
471+
<< "extent=" << e.size() << "D).";
472+
throw std::runtime_error(oss.str());
473+
}
474+
for (size_t i = 0; i < dim; ++i)
475+
{
476+
if (i != jd.value() && e[i] != dse[i])
477+
{
478+
throw std::runtime_error(
479+
"Joined array: Chunk extent on non-joined dimensions must "
480+
"be equivalent to dataset extents (Dimension on index " +
481+
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
482+
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
483+
}
484+
}
485+
}
486+
else
487+
{
488+
if (e.size() != dim || o.size() != dim)
489+
{
490+
std::ostringstream oss;
491+
oss << "Dimensionality of chunk ("
492+
<< "offset=" << o.size() << "D, "
493+
<< "extent=" << e.size() << "D) "
494+
<< "and record component (" << int(dim) << "D) "
495+
<< "do not match.";
496+
throw std::runtime_error(oss.str());
497+
}
498+
for (uint8_t i = 0; i < dim; ++i)
499+
if (dse[i] < o[i] + e[i])
500+
throw std::runtime_error(
501+
"Chunk does not reside inside dataset (Dimension on "
502+
"index " +
503+
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
504+
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
505+
}
463506
}
464507
} // namespace openPMD

‎src/backend/BaseRecordComponent.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,19 @@ bool BaseRecordComponent::constant() const
6565
return get().m_isConstant;
6666
}
6767

68+
std::optional<size_t> BaseRecordComponent::joinedDimension() const
69+
{
70+
auto &rc = get();
71+
if (rc.m_dataset.has_value())
72+
{
73+
return rc.m_dataset.value().joinedDimension;
74+
}
75+
else
76+
{
77+
return false;
78+
}
79+
}
80+
6881
ChunkTable BaseRecordComponent::availableChunks()
6982
{
7083
auto &rc = get();

0 commit comments

Comments
 (0)