Skip to content

Commit 8eb8cca

Browse files
franzpoeschelfranzpoeschel
committed
Main implementation
Use magic number instead of API call (impl)
1 parent d79ed2f commit 8eb8cca

File tree

12 files changed

+200
-30
lines changed

12 files changed

+200
-30
lines changed

‎include/openPMD/Dataset.hpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222

2323
#include "openPMD/Datatype.hpp"
2424

25+
#include <limits>
2526
#include <memory>
27+
#include <optional>
2628
#include <string>
2729
#include <type_traits>
2830
#include <vector>
@@ -37,6 +39,11 @@ class Dataset
3739
friend class RecordComponent;
3840

3941
public:
42+
enum : std::uint64_t
43+
{
44+
JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max()
45+
};
46+
4047
Dataset(Datatype, Extent, std::string options = "{}");
4148

4249
/**
@@ -53,5 +60,9 @@ class Dataset
5360
Datatype dtype;
5461
uint8_t rank;
5562
std::string options = "{}"; //!< backend-dependent JSON configuration
63+
64+
bool empty() const;
65+
66+
std::optional<size_t> joinedDimension() const;
5667
};
5768
} // namespace openPMD

‎include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ namespace openPMD
6060
{
6161
#if openPMD_HAVE_ADIOS2
6262

63+
std::optional<size_t> joinedDimension(adios2::Dims const &dims);
64+
6365
class ADIOS2IOHandler;
6466

6567
namespace detail
@@ -443,15 +445,24 @@ class ADIOS2IOHandlerImpl
443445
std::to_string(actualDim) + ")");
444446
}
445447
}
448+
auto joinedDim = joinedDimension(shape);
446449
for (unsigned int i = 0; i < actualDim; i++)
447450
{
448-
if (offset[i] + extent[i] > shape[i])
451+
if (!(joinedDim.has_value() && *joinedDim == i) &&
452+
offset[i] + extent[i] > shape[i])
449453
{
450454
throw std::runtime_error(
451455
"[ADIOS2] Dataset access out of bounds.");
452456
}
453457
}
454458

459+
if (joinedDim.has_value() && !offset.empty())
460+
{
461+
throw std::runtime_error(
462+
"[ADIOS2] Offset must be an empty vector in case of joined "
463+
"array.");
464+
}
465+
455466
var.SetSelection(
456467
{adios2::Dims(offset.begin(), offset.end()),
457468
adios2::Dims(extent.begin(), extent.end())});

‎include/openPMD/IO/IOTask.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
326326
Extent extent = {};
327327
Datatype dtype = Datatype::UNDEFINED;
328328
std::string options = "{}";
329+
std::optional<size_t> joinedDimension;
329330

330331
/** Warn about unused JSON paramters
331332
*

‎include/openPMD/RecordComponent.tpp‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -259,8 +259,17 @@ RecordComponent::storeChunk(T_ContiguousContainer &data, Offset o, Extent e)
259259
// default arguments
260260
// offset = {0u}: expand to right dim {0u, 0u, ...}
261261
Offset offset = o;
262-
if (o.size() == 1u && o.at(0) == 0u && dim > 1u)
263-
offset = Offset(dim, 0u);
262+
if (o.size() == 1u && o.at(0) == 0u)
263+
{
264+
if (joinedDimension().has_value())
265+
{
266+
offset.clear();
267+
}
268+
else if (dim > 1u)
269+
{
270+
offset = Offset(dim, 0u);
271+
}
272+
}
264273

265274
// extent = {-1u}: take full size
266275
Extent extent(dim, 1u);
@@ -303,6 +312,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
303312
dCreate.name = rc.m_name;
304313
dCreate.extent = getExtent();
305314
dCreate.dtype = getDatatype();
315+
dCreate.joinedDimension = joinedDimension();
306316
if (!rc.m_dataset.has_value())
307317
{
308318
throw error::WrongAPIUsage(

‎include/openPMD/backend/BaseRecordComponent.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ class BaseRecordComponent : virtual public Attributable
143143
*/
144144
bool constant() const;
145145

146+
std::optional<size_t> joinedDimension() const;
147+
146148
/**
147149
* Get data chunks that are available to be loaded from the backend.
148150
* Note that this is backend-dependent information and the returned

‎include/openPMD/backend/PatchRecordComponent.hpp‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
*/
2121
#pragma once
2222

23+
#include "openPMD/Error.hpp"
2324
#include "openPMD/auxiliary/ShareRawInternal.hpp"
2425
#include "openPMD/backend/BaseRecordComponent.hpp"
2526

@@ -115,6 +116,9 @@ class PatchRecordComponent : public BaseRecordComponent
115116
template <typename T>
116117
void store(uint64_t idx, T);
117118

119+
template <typename T>
120+
void store(T);
121+
118122
// clang-format off
119123
OPENPMD_private
120124
// clang-format on
@@ -244,4 +248,33 @@ inline void PatchRecordComponent::store(uint64_t idx, T data)
244248
auto &rc = get();
245249
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
246250
}
251+
252+
template <typename T>
253+
inline void PatchRecordComponent::store(T data)
254+
{
255+
Datatype dtype = determineDatatype<T>();
256+
if (dtype != getDatatype())
257+
{
258+
std::ostringstream oss;
259+
oss << "Datatypes of patch data (" << dtype << ") and dataset ("
260+
<< getDatatype() << ") do not match.";
261+
throw std::runtime_error(oss.str());
262+
}
263+
264+
if (!joinedDimension().has_value())
265+
{
266+
throw error::WrongAPIUsage(
267+
"[PatchRecordComponent::store] API call without explicit "
268+
"specification of index only allowed when a joined dimension is "
269+
"specified.");
270+
}
271+
272+
Parameter<Operation::WRITE_DATASET> dWrite;
273+
dWrite.offset = {};
274+
dWrite.extent = {1};
275+
dWrite.dtype = dtype;
276+
dWrite.data = std::make_shared<T>(data);
277+
auto &rc = get();
278+
rc.m_chunks.push(IOTask(this, std::move(dWrite)));
279+
}
247280
} // namespace openPMD

‎src/Dataset.cpp‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
* If not, see <http://www.gnu.org/licenses/>.
2020
*/
2121
#include "openPMD/Dataset.hpp"
22+
#include "openPMD/Error.hpp"
2223

2324
#include <cstddef>
2425
#include <iostream>
@@ -30,6 +31,9 @@ Dataset::Dataset(Datatype d, Extent e, std::string options_in)
3031
{
3132
// avoid initialization order issues
3233
rank = static_cast<uint8_t>(extent.size());
34+
// Call this in order to have early error message in case of wrong
35+
// specification of joined dimensions
36+
joinedDimension();
3337
}
3438

3539
Dataset::Dataset(Extent e) : Dataset(Datatype::UNDEFINED, std::move(e))
@@ -49,4 +53,41 @@ Dataset &Dataset::extend(Extent newExtents)
4953
extent = newExtents;
5054
return *this;
5155
}
56+
57+
bool Dataset::empty() const
58+
{
59+
auto jd = joinedDimension();
60+
for (size_t i = 0; i < extent.size(); ++i)
61+
{
62+
if (extent[i] == 0 && (!jd.has_value() || jd.value() != i))
63+
{
64+
return true;
65+
}
66+
}
67+
return false;
68+
}
69+
70+
std::optional<size_t> Dataset::joinedDimension() const
71+
{
72+
std::optional<size_t> res;
73+
for (size_t i = 0; i < extent.size(); ++i)
74+
{
75+
if (extent[i] == JOINED_DIMENSION)
76+
{
77+
if (res.has_value())
78+
{
79+
throw error::WrongAPIUsage(
80+
"Must specify JOINED_DIMENSION at most once (found at "
81+
"indices " +
82+
std::to_string(res.value()) + " and " + std::to_string(i) +
83+
")");
84+
}
85+
else
86+
{
87+
res = i;
88+
}
89+
}
90+
}
91+
return res;
92+
}
5293
} // namespace openPMD

‎src/IO/ADIOS/ADIOS2IOHandler.cpp‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,18 @@ namespace openPMD
6868

6969
#if openPMD_HAVE_ADIOS2
7070

71+
std::optional<size_t> joinedDimension(adios2::Dims const &dims)
72+
{
73+
for (size_t i = 0; i < dims.size(); ++i)
74+
{
75+
if (dims[i] == adios2::JoinedDim)
76+
{
77+
return i;
78+
}
79+
}
80+
return std::nullopt;
81+
}
82+
7183
#if openPMD_HAVE_MPI
7284

7385
ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl(
@@ -741,8 +753,11 @@ void ADIOS2IOHandlerImpl::createDataset(
741753
varName + "' remain unused:\n");
742754

743755
// cast from openPMD::Extent to adios2::Dims
744-
adios2::Dims const shape(
745-
parameters.extent.begin(), parameters.extent.end());
756+
adios2::Dims shape(parameters.extent.begin(), parameters.extent.end());
757+
if (auto jd = parameters.joinedDimension; jd.has_value())
758+
{
759+
shape[jd.value()] = adios2::JoinedDim;
760+
}
746761

747762
auto &fileData = getFileData(file, IfFileNotOpen::ThrowError);
748763

‎src/RecordComponent.cpp‎

Lines changed: 51 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
9595
}
9696
// if( d.extent.empty() )
9797
// throw std::runtime_error("Dataset extent must be at least 1D.");
98-
if (std::any_of(
99-
d.extent.begin(), d.extent.end(), [](Extent::value_type const &i) {
100-
return i == 0u;
101-
}))
98+
if (d.empty())
10299
return makeEmpty(std::move(d));
103100

104101
rc.m_isEmpty = false;
@@ -299,6 +296,7 @@ void RecordComponent::flush(
299296
dCreate.extent = getExtent();
300297
dCreate.dtype = getDatatype();
301298
dCreate.options = rc.m_dataset.value().options;
299+
dCreate.joinedDimension = joinedDimension();
302300
IOHandler()->enqueue(IOTask(this, dCreate));
303301
}
304302
}
@@ -474,22 +472,57 @@ void RecordComponent::verifyChunk(
474472
uint8_t dim = getDimensionality();
475473
Extent dse = getExtent();
476474

477-
if (e.size() != dim || o.size() != dim)
475+
if (auto jd = joinedDimension(); jd.has_value())
478476
{
479-
std::ostringstream oss;
480-
oss << "Dimensionality of chunk ("
481-
<< "offset=" << o.size() << "D, "
482-
<< "extent=" << e.size() << "D) "
483-
<< "and record component (" << int(dim) << "D) "
484-
<< "do not match.";
485-
throw std::runtime_error(oss.str());
477+
if (o.size() != 0)
478+
{
479+
std::ostringstream oss;
480+
oss << "Joined array: Must specify an empty offset (given: "
481+
<< "offset=" << o.size() << "D, "
482+
<< "extent=" << e.size() << "D).";
483+
throw std::runtime_error(oss.str());
484+
}
485+
if (e.size() != dim)
486+
{
487+
std::ostringstream oss;
488+
oss << "Joined array: Dimensionalities of chunk extent and dataset "
489+
"extent must be equivalent (given: "
490+
<< "offset=" << o.size() << "D, "
491+
<< "extent=" << e.size() << "D).";
492+
throw std::runtime_error(oss.str());
493+
}
494+
for (size_t i = 0; i < dim; ++i)
495+
{
496+
if (i != jd.value() && e[i] != dse[i])
497+
{
498+
throw std::runtime_error(
499+
"Joined array: Chunk extent on non-joined dimensions must "
500+
"be equivalent to dataset extents (Dimension on index " +
501+
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
502+
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
503+
}
504+
}
505+
}
506+
else
507+
{
508+
if (e.size() != dim || o.size() != dim)
509+
{
510+
std::ostringstream oss;
511+
oss << "Dimensionality of chunk ("
512+
<< "offset=" << o.size() << "D, "
513+
<< "extent=" << e.size() << "D) "
514+
<< "and record component (" << int(dim) << "D) "
515+
<< "do not match.";
516+
throw std::runtime_error(oss.str());
517+
}
518+
for (uint8_t i = 0; i < dim; ++i)
519+
if (dse[i] < o[i] + e[i])
520+
throw std::runtime_error(
521+
"Chunk does not reside inside dataset (Dimension on "
522+
"index " +
523+
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
524+
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
486525
}
487-
for (uint8_t i = 0; i < dim; ++i)
488-
if (dse[i] < o[i] + e[i])
489-
throw std::runtime_error(
490-
"Chunk does not reside inside dataset (Dimension on index " +
491-
std::to_string(i) + ". DS: " + std::to_string(dse[i]) +
492-
" - Chunk: " + std::to_string(o[i] + e[i]) + ")");
493526
}
494527

495528
namespace

‎src/backend/BaseRecordComponent.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,19 @@ bool BaseRecordComponent::constant() const
6565
return get().m_isConstant;
6666
}
6767

68+
std::optional<size_t> BaseRecordComponent::joinedDimension() const
69+
{
70+
auto &rc = get();
71+
if (rc.m_dataset.has_value())
72+
{
73+
return rc.m_dataset.value().joinedDimension();
74+
}
75+
else
76+
{
77+
return false;
78+
}
79+
}
80+
6881
ChunkTable BaseRecordComponent::availableChunks()
6982
{
7083
auto &rc = get();

0 commit comments

Comments
 (0)