Skip to content

Commit ab29e02

Browse files
franzpoeschelfranzpoeschel
committed
Add template mode to JSON backend
1 parent 0b4bb53 commit ab29e02

File tree

4 files changed

+114
-46
lines changed

4 files changed

+114
-46
lines changed

‎CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -814,6 +814,7 @@ set(openPMD_EXAMPLE_NAMES
814814
10_streaming_read
815815
12_span_write
816816
13_write_dynamic_configuration
817+
14_json_template
817818
)
818819
set(openPMD_PYTHON_EXAMPLE_NAMES
819820
2_read_serial

‎examples/14_json_template.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#include <openPMD/openPMD.hpp>
2+
3+
int main()
4+
{
5+
openPMD::Series writeTemplate(
6+
"../samples/jsonTemplate.json",
7+
openPMD::Access::CREATE,
8+
R"(iteration_encoding = "variable_based")");
9+
auto iteration = writeTemplate.writeIterations()[0];
10+
11+
auto temperature =
12+
iteration.meshes["temperature"][openPMD::RecordComponent::SCALAR];
13+
temperature.resetDataset({openPMD::Datatype::FLOAT, {5, 5}});
14+
}

‎include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,14 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
224224
// files that have logically, but not physically been written to
225225
std::unordered_set<File> m_dirty;
226226

227+
enum class IOMode
228+
{
229+
Dataset,
230+
Template
231+
};
232+
233+
IOMode m_mode = IOMode::Template;
234+
227235
// HELPER FUNCTIONS
228236

229237
// will use the IOHandler to retrieve the correct directory
@@ -269,7 +277,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
269277

270278
static nlohmann::json initializeNDArray(Extent const &extent);
271279

272-
static Extent getExtent(nlohmann::json &j);
280+
static Extent getExtent(nlohmann::json &j, IOMode);
273281

274282
// remove single '/' in the beginning and end of a string
275283
static std::string removeSlashes(std::string);

‎src/IO/JSON/JSONIOHandlerImpl.cpp

Lines changed: 90 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp"
2323
#include "openPMD/Datatype.hpp"
2424
#include "openPMD/DatatypeHelpers.hpp"
25+
#include "openPMD/Error.hpp"
2526
#include "openPMD/auxiliary/Filesystem.hpp"
2627
#include "openPMD/auxiliary/Memory.hpp"
2728
#include "openPMD/auxiliary/StringManip.hpp"
@@ -200,20 +201,30 @@ void JSONIOHandlerImpl::createDataset(
200201
setAndGetFilePosition(writable, name);
201202
auto &dset = jsonVal[name];
202203
dset["datatype"] = datatypeToString(parameter.dtype);
203-
switch (parameter.dtype)
204+
205+
switch (m_mode)
204206
{
205-
case Datatype::CFLOAT:
206-
case Datatype::CDOUBLE:
207-
case Datatype::CLONG_DOUBLE: {
208-
auto complexExtent = parameter.extent;
209-
complexExtent.push_back(2);
210-
dset["data"] = initializeNDArray(complexExtent);
207+
case IOMode::Dataset: {
208+
auto extent = parameter.extent;
209+
switch (parameter.dtype)
210+
{
211+
case Datatype::CFLOAT:
212+
case Datatype::CDOUBLE:
213+
case Datatype::CLONG_DOUBLE: {
214+
extent.push_back(2);
215+
break;
216+
}
217+
default:
218+
break;
219+
}
220+
dset["data"] = initializeNDArray(extent);
211221
break;
212222
}
213-
default:
214-
dset["data"] = initializeNDArray(parameter.extent);
223+
case IOMode::Template:
224+
dset["extent"] = parameter.extent;
215225
break;
216226
}
227+
217228
writable->written = true;
218229
m_dirty.emplace(file);
219230
}
@@ -254,7 +265,7 @@ void JSONIOHandlerImpl::extendDataset(
254265

255266
try
256267
{
257-
auto datasetExtent = getExtent(j);
268+
auto datasetExtent = getExtent(j, m_mode);
258269
VERIFY_ALWAYS(
259270
datasetExtent.size() == parameters.extent.size(),
260271
"[JSON] Cannot change dimensionality of a dataset")
@@ -271,27 +282,33 @@ void JSONIOHandlerImpl::extendDataset(
271282
throw std::runtime_error(
272283
"[JSON] The specified location contains no valid dataset");
273284
}
274-
switch (stringToDatatype(j["datatype"].get<std::string>()))
285+
286+
switch (m_mode)
275287
{
276-
case Datatype::CFLOAT:
277-
case Datatype::CDOUBLE:
278-
case Datatype::CLONG_DOUBLE: {
279-
// @todo test complex resizing
280-
auto complexExtent = parameters.extent;
281-
complexExtent.push_back(2);
282-
nlohmann::json newData = initializeNDArray(complexExtent);
283-
nlohmann::json &oldData = j["data"];
284-
mergeInto(newData, oldData);
285-
j["data"] = newData;
288+
case IOMode::Template:
289+
j["extent"] = parameters.extent;
286290
break;
287-
}
288-
default:
289-
nlohmann::json newData = initializeNDArray(parameters.extent);
291+
case IOMode::Dataset: {
292+
auto extent = parameters.extent;
293+
switch (stringToDatatype(j["datatype"].get<std::string>()))
294+
{
295+
case Datatype::CFLOAT:
296+
case Datatype::CDOUBLE:
297+
case Datatype::CLONG_DOUBLE: {
298+
extent.push_back(2);
299+
break;
300+
}
301+
default:
302+
// nothing to do
303+
break;
304+
}
305+
nlohmann::json newData = initializeNDArray(extent);
290306
nlohmann::json &oldData = j["data"];
291307
mergeInto(newData, oldData);
292308
j["data"] = newData;
293309
break;
294310
}
311+
}
295312
writable->written = true;
296313
}
297314

@@ -583,7 +600,7 @@ void JSONIOHandlerImpl::openDataset(
583600

584601
*parameters.dtype =
585602
Datatype(stringToDatatype(datasetJson["datatype"].get<std::string>()));
586-
*parameters.extent = getExtent(datasetJson);
603+
*parameters.extent = getExtent(datasetJson, m_mode);
587604
writable->written = true;
588605
}
589606

@@ -762,6 +779,15 @@ void JSONIOHandlerImpl::writeDataset(
762779
m_handler->m_backendAccess != Access::READ_ONLY,
763780
"[JSON] Cannot write data in read-only mode.");
764781

782+
switch (m_mode)
783+
{
784+
case IOMode::Template:
785+
throw error::WrongAPIUsage(
786+
"Cannot write chunks in Template mode of JSON backend.");
787+
case IOMode::Dataset:
788+
break;
789+
}
790+
765791
auto pos = setAndGetFilePosition(writable);
766792
auto file = refreshFileFromParent(writable);
767793
auto &j = obtainJsonContents(writable);
@@ -805,6 +831,15 @@ void JSONIOHandlerImpl::writeAttribute(
805831
void JSONIOHandlerImpl::readDataset(
806832
Writable *writable, Parameter<Operation::READ_DATASET> &parameters)
807833
{
834+
switch (m_mode)
835+
{
836+
case IOMode::Template:
837+
throw error::WrongAPIUsage(
838+
"Cannot write chunks in Template mode of JSON backend.");
839+
case IOMode::Dataset:
840+
break;
841+
}
842+
808843
refreshFileFromParent(writable);
809844
setAndGetFilePosition(writable);
810845
auto &j = obtainJsonContents(writable);
@@ -1043,25 +1078,34 @@ nlohmann::json JSONIOHandlerImpl::initializeNDArray(Extent const &extent)
10431078
return *accum_ptr;
10441079
}
10451080

1046-
Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j)
1081+
Extent JSONIOHandlerImpl::getExtent(nlohmann::json &j, IOMode mode)
10471082
{
10481083
Extent res;
1049-
nlohmann::json *ptr = &j["data"];
1050-
while (ptr->is_array())
1084+
switch (mode)
10511085
{
1052-
res.push_back(ptr->size());
1053-
ptr = &(*ptr)[0];
1086+
case IOMode::Dataset: {
1087+
nlohmann::json *ptr = &j["data"];
1088+
while (ptr->is_array())
1089+
{
1090+
res.push_back(ptr->size());
1091+
ptr = &(*ptr)[0];
1092+
}
1093+
switch (stringToDatatype(j["datatype"].get<std::string>()))
1094+
{
1095+
case Datatype::CFLOAT:
1096+
case Datatype::CDOUBLE:
1097+
case Datatype::CLONG_DOUBLE:
1098+
// the last "dimension" is only the two entries for the complex
1099+
// number, so remove that again
1100+
res.erase(res.end() - 1);
1101+
break;
1102+
default:
1103+
break;
1104+
}
10541105
}
1055-
switch (stringToDatatype(j["datatype"].get<std::string>()))
1056-
{
1057-
case Datatype::CFLOAT:
1058-
case Datatype::CDOUBLE:
1059-
case Datatype::CLONG_DOUBLE:
1060-
// the last "dimension" is only the two entries for the complex
1061-
// number, so remove that again
1062-
res.erase(res.end() - 1);
1063-
break;
1064-
default:
1106+
break;
1107+
case IOMode::Template:
1108+
res = j["extent"].get<Extent>();
10651109
break;
10661110
}
10671111
return res;
@@ -1258,8 +1302,8 @@ bool JSONIOHandlerImpl::isDataset(nlohmann::json const &j)
12581302
{
12591303
return false;
12601304
}
1261-
auto i = j.find("data");
1262-
return i != j.end() && i.value().is_array();
1305+
auto i = j.find("datatype");
1306+
return i != j.end() && i.value().is_string();
12631307
}
12641308

12651309
bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator it)
@@ -1270,8 +1314,9 @@ bool JSONIOHandlerImpl::isGroup(nlohmann::json::const_iterator it)
12701314
{
12711315
return false;
12721316
}
1273-
auto i = j.find("data");
1274-
return i == j.end() || !i.value().is_array();
1317+
1318+
auto i = j.find("datatype");
1319+
return i == j.end() || !i.value().is_string();
12751320
}
12761321

12771322
template <typename Param>
@@ -1284,7 +1329,7 @@ void JSONIOHandlerImpl::verifyDataset(
12841329

12851330
try
12861331
{
1287-
auto datasetExtent = getExtent(j);
1332+
auto datasetExtent = getExtent(j, m_mode);
12881333
VERIFY_ALWAYS(
12891334
datasetExtent.size() == parameters.extent.size(),
12901335
"[JSON] Read/Write request does not fit the dataset's dimension");

0 commit comments

Comments
 (0)