Use llama for particle frame and shared memory DataBox layout

Also support LLAMA frames in the IO.

Author: bernhardmgruber

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "thirdParty/llama"]
+	path = thirdParty/llama
+	url = https://github.com/alpaka-group/llama

include/picongpu/algorithms/Set.hpp

@@ -33,9 +33,9 @@ namespace picongpu
         }
 
         template<typename Dst, typename T_Worker>
-        HDINLINE void operator()(T_Worker const&, Dst& dst) const
+        HDINLINE void operator()(T_Worker const&, Dst&& dst) const
         {
-            dst = value;
+            std::forward<Dst>(dst) = value;
         }
 
     private:

include/picongpu/algorithms/Velocity.hpp

@@ -28,7 +28,7 @@ namespace picongpu
     struct Velocity
     {
         template<typename MomType, typename MassType>
-        HDINLINE MomType operator()(const MomType mom, const MassType mass0)
+        HDINLINE auto operator()(const MomType mom, const MassType mass0)
         {
             const float_X rc2 = MUE0_EPS0;
             const float_X m0_2 = mass0 * mass0;

include/picongpu/fields/FieldJ.kernel

@@ -131,7 +131,8 @@ namespace picongpu
                 // The rest uses normal weighting
                 const float_X weighting = particle[weighting_];
                 Velocity velocity;
-                const float3_X vel = velocity(particle[momentum_], attribute::getMass(weighting, particle));
+                const float3_X vel
+                    = velocity(static_cast<float3_X>(particle[momentum_]), attribute::getMass(weighting, particle));
                 auto fieldJShiftToParticle = jBox.shift(localCell);
                 ParticleAlgo perParticle;
                 perParticle(worker, fieldJShiftToParticle, pos, vel, charge, m_deltaTime);

include/picongpu/fields/FieldTmp.kernel

@@ -94,7 +94,9 @@ namespace picongpu
             if(!forEachParticle.hasParticles())
                 return;
 
-            auto cachedVal = CachedBox::create<0, typename T_TmpBox::ValueType>(worker, T_BlockDescription{});
+            auto cachedVal = CachedBox::create<0, SharedDataBoxMemoryLayout, typename T_TmpBox::ValueType>(
+                worker,
+                T_BlockDescription{});
             Set<typename T_TmpBox::ValueType> set(float_X(0.0));
 
             auto collective = makeThreadCollective<T_BlockDescription>();

include/picongpu/fields/MaxwellSolver/AddCurrentDensity.kernel

@@ -22,6 +22,8 @@
 
 #include "picongpu/simulation_defines.hpp"
 
+#include "picongpu/param/memory.param"
+
 #include <pmacc/dimensions/SuperCellDescription.hpp>
 #include <pmacc/lockstep.hpp>
 #include <pmacc/mappings/threads/ThreadCollective.hpp>
@@ -68,7 +70,9 @@ namespace picongpu::fields::maxwellSolver
 
             constexpr uint32_t cellsPerSuperCell = pmacc::math::CT::volume<SuperCellSize>::type::value;
 
-            auto cachedJ = CachedBox::create<0, typename FieldJ::DataBoxType::ValueType>(worker, BlockArea());
+            auto cachedJ = CachedBox::create<0, SharedDataBoxMemoryLayout, typename FieldJ::DataBoxType::ValueType>(
+                worker,
+                BlockArea());
 
             pmacc::math::operation::Assign assign;
             DataSpace<simDim> const block(

include/picongpu/fields/MaxwellSolver/FDTD/FDTDBase.kernel

@@ -179,7 +179,9 @@ namespace picongpu
                         auto srcFieldBlock = srcField.shift(beginCellIdx);
                         auto cacheStencilArea = makeThreadCollective<StencilCfg>();
                         auto cachedSrcField
-                            = CachedBox::create<0u, typename T_SrcBox::ValueType>(worker, StencilCfg{});
+                            = CachedBox::create<0u, SharedDataBoxMemoryLayout, typename T_SrcBox::ValueType>(
+                                worker,
+                                StencilCfg{});
                         cacheStencilArea(worker, assign, cachedSrcField, srcFieldBlock);
 
                         worker.sync();

include/picongpu/fields/currentDeposition/Cache.hpp

@@ -48,15 +48,11 @@ namespace picongpu
                  */
                 template<typename T_BlockDescription, typename T_Worker, typename T_FieldBox>
                 DINLINE static auto create(T_Worker const& worker, T_FieldBox const& fieldBox)
-#if(!BOOST_COMP_CLANG)
-                    -> decltype(CachedBox::create<0u, typename T_FieldBox::ValueType>(
-                        worker,
-                        std::declval<T_BlockDescription>()))
-#endif
                 {
                     using ValueType = typename T_FieldBox::ValueType;
                     /* this memory is used by all virtual blocks */
-                    auto cache = CachedBox::create<0u, ValueType>(worker, T_BlockDescription{});
+                    auto cache
+                        = CachedBox::create<0u, SharedDataBoxMemoryLayout, ValueType>(worker, T_BlockDescription{});
 
                     Set<ValueType> set(ValueType::create(0.0_X));
                     auto collectiveFill = makeThreadCollective<T_BlockDescription>();
@@ -90,9 +86,6 @@ namespace picongpu
                  */
                 template<typename T_BlockDescription, typename T_Worker, typename T_FieldBox>
                 DINLINE static auto create([[maybe_unused]] T_Worker const& worker, T_FieldBox const& fieldBox)
-#if(!BOOST_COMP_CLANG)
-                    -> T_FieldBox
-#endif
                 {
                     return fieldBox;
                 }

include/picongpu/fields/incidentField/Solver.hpp

@@ -283,8 +283,9 @@ namespace picongpu
                     using IntVector = pmacc::math::Vector<int, simDim>;
                     auto const beginLocalUserIdx
                         = Index{math::max(IntVector{beginUserIdx - totalCellOffset}, IntVector::create(0))};
-                    auto const endLocalUserIdx
-                        = Index{math::min(IntVector{endUserIdx - totalCellOffset}, IntVector{localDomain.size})};
+                    auto const endLocalUserIdx = Index{math::min(
+                        IntVector{endUserIdx - totalCellOffset},
+                        static_cast<const IntVector&>(localDomain.size))};
 
                     // Check if we have any active cells in the local domain
                     bool areAnyCellsInLocalDomain = true;

include/picongpu/param/memory.param

@@ -114,4 +114,19 @@ namespace picongpu
      */
     constexpr bool fieldTmpSupportGatherCommunication = true;
 
+    struct ParticleFrameMemoryLayout
+        : llama::mapping::BindSoA<llama::mapping::Blobs::Single, llama::mapping::SubArrayAlignment::Align>
+    {
+        inline static constexpr bool splitVector = false;
+    };
+
+    struct ParticleFrameMemoryLayoutOpenPMD : llama::mapping::BindSoA<llama::mapping::Blobs::OnePerField>
+    {
+        inline static constexpr bool splitVector = false;
+    };
+
+    struct SharedDataBoxMemoryLayout : llama::mapping::BindAoS<>
+    {
+        inline static constexpr bool splitVector = false;
+    };
 } // namespace picongpu