Skip to content

Commit 3acaf12

Browse files
bernhardmgruberbernhardmgruber
committed
Rewrite IO with LLAMA
1 parent 8790d1b commit 3acaf12

File tree

7 files changed

+110
-126
lines changed

7 files changed

+110
-126
lines changed

‎include/picongpu/plugins/openPMD/WriteSpecies.hpp‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -113,15 +113,13 @@ namespace picongpu
113113
{
114114
/* malloc host memory */
115115
log<picLog::INPUT_OUTPUT>("openPMD: (begin) malloc host memory: %1%") % name;
116-
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, MallocHostMemory<bmpl::_1>> mallocMem;
117-
mallocMem(hostFrame, myNumParticles);
116+
mallocFrameMemory(hostFrame);
118117
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) malloc host memory: %1%") % name;
119118
}
120119

121120
void free(openPMDFrameType& hostFrame) override
122121
{
123-
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, FreeHostMemory<bmpl::_1>> freeMem;
124-
freeMem(hostFrame);
122+
freeFrameMemory(hostFrame);
125123
}
126124

127125

@@ -182,16 +180,13 @@ namespace picongpu
182180
void malloc(std::string name, openPMDFrameType& mappedFrame, uint64_cu const myNumParticles) override
183181
{
184182
log<picLog::INPUT_OUTPUT>("openPMD: (begin) malloc mapped memory: %1%") % name;
185-
/*malloc mapped memory*/
186-
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, MallocMappedMemory<bmpl::_1>> mallocMem;
187-
mallocMem(mappedFrame, myNumParticles);
183+
mallocMappedFrameMemory(mappedFrame);
188184
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) malloc mapped memory: %1%") % name;
189185
}
190186

191187
void free(openPMDFrameType& mappedFrame) override
192188
{
193-
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, FreeMappedMemory<bmpl::_1>> freeMem;
194-
freeMem(mappedFrame);
189+
freeMappedFrameMemory(mappedFrame);
195190
}
196191

197192
void prepare(std::string name, openPMDFrameType& mappedFrame, RunParameters rp) override
@@ -247,7 +242,7 @@ namespace picongpu
247242
using NewParticleDescription =
248243
typename ReplaceValueTypeSeq<ParticleDescription, ParticleNewAttributeList>::type;
249244

250-
using openPMDFrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
245+
using openPMDFrameType = Frame<llama::dyn, NewParticleDescription>;
251246

252247
void setParticleAttributes(
253248
::openPMD::ParticleSpecies& record,
@@ -361,14 +356,12 @@ namespace picongpu
361356
{
362357
case WriteSpeciesStrategy::ADIOS:
363358
{
364-
using type = StrategyADIOS<openPMDFrameType, RunParameters_T>;
365-
strategy = std::unique_ptr<AStrategy>(dynamic_cast<AStrategy*>(new type));
359+
strategy = std::make_unique<StrategyADIOS<openPMDFrameType, RunParameters_T>>();
366360
break;
367361
}
368362
case WriteSpeciesStrategy::HDF5:
369363
{
370-
using type = StrategyHDF5<openPMDFrameType, RunParameters_T>;
371-
strategy = std::unique_ptr<AStrategy>(dynamic_cast<AStrategy*>(new type));
364+
strategy = std::make_unique<StrategyHDF5<openPMDFrameType, RunParameters_T>>();
372365
break;
373366
}
374367
}

‎include/picongpu/plugins/openPMD/restart/LoadParticleAttributesFromOpenPMD.hpp‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,15 @@ namespace picongpu
6868
{
6969
using Identifier = T_Identifier;
7070
using ValueType = typename pmacc::traits::Resolve<Identifier>::type::type;
71-
const uint32_t components = GetNComponents<ValueType>::value;
71+
constexpr uint32_t components = GetNComponents<ValueType>::value;
7272
using ComponentType = typename GetComponentsType<ValueType>::type;
7373
OpenPMDName<Identifier> openPMDName;
7474

7575
log<picLog::INPUT_OUTPUT>("openPMD: ( begin ) load species attribute: %1%") % openPMDName();
7676

7777
const std::string name_lookup[] = {"x", "y", "z"};
7878

79+
// TODO(bgruber): make this a std::shared_ptr<ComponentType[]> with openPMD 0.15
7980
std::shared_ptr<ComponentType> loadBfr;
8081
if(elements > 0)
8182
{
@@ -90,7 +91,6 @@ namespace picongpu
9091
::openPMD::RecordComponent rc
9192
= components > 1 ? record[name_lookup[n]] : record[::openPMD::RecordComponent::SCALAR];
9293

93-
ValueType* dataPtr = frame.getIdentifier(Identifier()).getPointer();
9494

9595
if(elements > 0)
9696
{
@@ -119,12 +119,15 @@ namespace picongpu
119119
"%3%")
120120
% elements % globalNumElements % openPMDName();
121121

122-
/* copy component from temporary array to array of structs */
122+
/* copy component from temporary array to array of structs */
123123
#pragma omp parallel for simd
124124
for(size_t i = 0; i < elements; ++i)
125125
{
126-
ComponentType* ref = &reinterpret_cast<ComponentType*>(dataPtr)[i * components + n];
127-
*ref = loadBfr.get()[i];
126+
auto& attrib = frame[i][Identifier{}];
127+
if constexpr(components == 1)
128+
attrib = loadBfr.get()[i];
129+
else
130+
reinterpret_cast<ComponentType*>(&attrib)[n] = loadBfr.get()[i];
128131
}
129132
}
130133

‎include/picongpu/plugins/openPMD/restart/LoadSpecies.hpp‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ namespace picongpu
7474
using NewParticleDescription =
7575
typename ReplaceValueTypeSeq<ParticleDescription, ParticleNewAttributeList>::type;
7676

77-
using openPMDFrameType = Frame<OperatorCreateVectorBox, NewParticleDescription>;
77+
using openPMDFrameType = Frame<llama::dyn, NewParticleDescription>;
7878

7979
/** Load species from openPMD checkpoint storage
8080
*
@@ -138,9 +138,8 @@ namespace picongpu
138138
// memory is visible on host and device
139139
openPMDFrameType mappedFrame;
140140
log<picLog::INPUT_OUTPUT>("openPMD: malloc mapped memory: %1%") % speciesName;
141-
/*malloc mapped memory*/
142-
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, MallocMappedMemory<bmpl::_1>> mallocMem;
143-
mallocMem(mappedFrame, totalNumParticles);
141+
142+
mallocMappedFrameMemory(mappedFrame);
144143

145144
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, LoadParticleAttributesFromOpenPMD<bmpl::_1>>
146145
loadAttributes;
@@ -158,9 +157,7 @@ namespace picongpu
158157
*(params->cellDescription),
159158
picLog::INPUT_OUTPUT());
160159

161-
/*free host memory*/
162-
meta::ForEach<typename openPMDFrameType::ValueTypeSeq, FreeMappedMemory<bmpl::_1>> freeMem;
163-
freeMem(mappedFrame);
160+
freeMappedFrameMemory(mappedFrame);
164161
}
165162
log<picLog::INPUT_OUTPUT>("openPMD: ( end ) load species: %1%") % speciesName;
166163
}

‎include/picongpu/plugins/openPMD/writer/ParticleAttribute.hpp‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ namespace picongpu
6464
{
6565
using Identifier = T_Identifier;
6666
using ValueType = typename pmacc::traits::Resolve<Identifier>::type::type;
67-
const uint32_t components = GetNComponents<ValueType>::value;
67+
constexpr uint32_t components = GetNComponents<ValueType>::value;
6868
using ComponentType = typename GetComponentsType<ValueType>::type;
6969

7070
OpenPMDName<T_Identifier> openPMDName;
@@ -93,7 +93,7 @@ namespace picongpu
9393
::openPMD::RecordComponent recordComponent
9494
= components > 1 ? record[name_lookup[d]] : record[::openPMD::MeshRecordComponent::SCALAR];
9595

96-
std::string datasetName = components > 1 ? baseName + "/" + name_lookup[d] : baseName;
96+
const std::string datasetName = components > 1 ? baseName + "/" + name_lookup[d] : baseName;
9797
params->initDataset<DIM1>(recordComponent, openPMDType, {globalElements}, datasetName);
9898

9999
if(unit.size() >= (d + 1))
@@ -107,7 +107,6 @@ namespace picongpu
107107
continue;
108108
}
109109

110-
ValueType* dataPtr = frame.getIdentifier(Identifier()).getPointer(); // can be moved up?
111110
// ask openPMD to create a buffer for us
112111
// in some backends (ADIOS2), this allows avoiding memcopies
113112
auto span = storeChunkSpan<ComponentType>(
@@ -132,7 +131,11 @@ namespace picongpu
132131
#pragma omp parallel for simd
133132
for(size_t i = 0; i < elements; ++i)
134133
{
135-
span[i] = reinterpret_cast<ComponentType*>(dataPtr)[d + i * components];
134+
const auto attrib = frame[i][Identifier{}];
135+
if constexpr(components == 1)
136+
span[i] = attrib;
137+
else
138+
span[i] = reinterpret_cast<const ComponentType*>(&attrib)[d];
136139
}
137140

138141
flushSeries(*params->openPMDSeries, PreferredFlushTarget::Disk);

‎include/picongpu/plugins/output/WriteSpeciesCommon.hpp‎

Lines changed: 34 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -31,51 +31,48 @@ namespace picongpu
3131
{
3232
using namespace pmacc;
3333

34-
35-
template<typename T_Type>
36-
struct MallocMappedMemory
34+
template<typename Frame>
35+
void mallocMappedFrameMemory(Frame& frame)
3736
{
38-
template<typename ValueType>
39-
HINLINE void operator()(ValueType& v1, const size_t size) const
40-
{
41-
using type = typename pmacc::traits::Resolve<T_Type>::type::type;
42-
43-
bool isMappedMemorySupported = alpaka::hasMappedBufSupport<::alpaka::Pltf<cupla::AccDev>>;
44-
45-
PMACC_VERIFY_MSG(isMappedMemorySupported, "Device must support mapped memory!");
37+
constexpr bool isMappedMemorySupported = alpaka::hasMappedBufSupport<::alpaka::Pltf<cupla::AccDev>>;
38+
PMACC_VERIFY_MSG(isMappedMemorySupported, "Device must support mapped memory!");
4639

47-
type* ptr = nullptr;
40+
int i = 0;
41+
for(std::byte*& ptr : frame.blobs())
42+
{
43+
const auto size = frame.blobSize(i);
4844
if(size != 0)
4945
{
5046
// Memory is automatically mapped to the device if supported.
51-
CUDA_CHECK(cuplaMallocHost((void**) &ptr, size * sizeof(type)));
47+
CUDA_CHECK(cuplaMallocHost((void**) &ptr, size));
5248
}
53-
v1.getIdentifier(T_Type()) = VectorDataBox<type>(ptr);
49+
else
50+
ptr = nullptr;
51+
i++;
5452
}
55-
};
53+
}
5654

5755
/** allocate memory on host
5856
*
5957
* This functor use `new[]` to allocate memory
6058
*/
61-
template<typename T_Attribute>
62-
struct MallocHostMemory
59+
template<typename Frame>
60+
void mallocFrameMemory(Frame& frame)
6361
{
64-
template<typename ValueType>
65-
HINLINE void operator()(ValueType& v1, const size_t size) const
66-
{
67-
using Attribute = T_Attribute;
68-
using type = typename pmacc::traits::Resolve<Attribute>::type::type;
62+
constexpr bool isMappedMemorySupported = alpaka::hasMappedBufSupport<::alpaka::Pltf<cupla::AccDev>>;
63+
PMACC_VERIFY_MSG(isMappedMemorySupported, "Device must support mapped memory!");
6964

70-
type* ptr = nullptr;
65+
int i = 0;
66+
for(std::byte*& ptr : frame.blobs())
67+
{
68+
const auto size = frame.blobSize(i);
7169
if(size != 0)
72-
{
73-
ptr = new type[size];
74-
}
75-
v1.getIdentifier(Attribute()) = VectorDataBox<type>(ptr);
70+
ptr = new std::byte[size];
71+
else
72+
ptr = nullptr;
73+
i++;
7674
}
77-
};
78-
75+
}
7976

8077
/** copy species to host memory
8178
*
@@ -94,42 +91,18 @@ namespace picongpu
9491
}
9592
};
9693

97-
template<typename T_Type>
98-
struct FreeMappedMemory
94+
template<typename Frame>
95+
void freeMappedFrameMemory(Frame& frame)
9996
{
100-
template<typename ValueType>
101-
HINLINE void operator()(ValueType& value) const
102-
{
103-
auto* ptr = value.getIdentifier(T_Type()).getPointer();
97+
for(auto* ptr : frame.blobs())
10498
if(ptr != nullptr)
105-
{
10699
CUDA_CHECK(cuplaFreeHost(ptr));
107-
}
108-
}
109-
};
100+
}
110101

111-
//! Free memory
112-
template<typename T_Attribute>
113-
struct FreeHostMemory
102+
template<typename Frame>
103+
void freeFrameMemory(Frame& frame)
114104
{
115-
template<typename ValueType>
116-
HINLINE void operator()(ValueType& value) const
117-
{
118-
using Attribute = T_Attribute;
119-
120-
auto* ptr = value.getIdentifier(Attribute()).getPointer();
105+
for(auto* ptr : frame.blobs())
121106
delete[] ptr;
122-
}
123-
};
124-
125-
/*functor to create a pair for a MapTuple map*/
126-
struct OperatorCreateVectorBox
127-
{
128-
template<typename InType>
129-
struct apply
130-
{
131-
typedef bmpl::pair<InType, pmacc::VectorDataBox<typename pmacc::traits::Resolve<InType>::type::type>> type;
132-
};
133-
};
134-
107+
}
135108
} // namespace picongpu

‎include/pmacc/particles/memory/buffers/ParticlesBuffer.hpp‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,7 @@ namespace pmacc
103103
*
104104
* a group of particles is stored as frame
105105
*/
106-
using FrameType = Frame<
107-
OperatorCreatePairStaticArray<pmacc::math::CT::volume<SuperCellSize>::type::value>,
108-
FrameDescription>;
106+
using FrameType = Frame<pmacc::math::CT::volume<SuperCellSize>::type::value, FrameDescription>;
109107

110108
using FrameDescriptionBorder =
111109
typename ReplaceValueTypeSeq<T_ParticleDescription, ParticleAttributeListBorder>::type;
@@ -115,7 +113,7 @@ namespace pmacc
115113
* - each frame contains only one particle
116114
* - local administration attributes of a particle are removed
117115
*/
118-
using FrameTypeBorder = Frame<OperatorCreatePairStaticArray<1U>, FrameDescriptionBorder>;
116+
using FrameTypeBorder = Frame<1, FrameDescriptionBorder>;
119117

120118
using SuperCellType = SuperCell<FrameType>;
121119

0 commit comments

Comments
 (0)