From 3d27c8e7bc007187c002985581a14aa14ce789c2 Mon Sep 17 00:00:00 2001 From: Alexandre Hoffmann Date: Wed, 28 May 2025 17:33:32 +0200 Subject: [PATCH 1/3] feurst --- include/samurai/algorithm/update.hpp | 87 ++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/include/samurai/algorithm/update.hpp b/include/samurai/algorithm/update.hpp index da0b2bec9..4b7acfdec 100644 --- a/include/samurai/algorithm/update.hpp +++ b/include/samurai/algorithm/update.hpp @@ -94,6 +94,7 @@ namespace samurai void update_ghost_mr(Field& field, Fields&... other_fields) { using mesh_id_t = typename Field::mesh_t::mesh_id_t; + using Field_value_t = typename Field::value_type; constexpr std::size_t pred_order = Field::mesh_t::config::prediction_order; times::timers.start("ghost update"); @@ -104,20 +105,98 @@ namespace samurai mpi::communicator world; auto min_level = mpi::all_reduce(world, mesh[mesh_id_t::reference].min_level(), mpi::minimum()); auto max_level = mpi::all_reduce(world, mesh[mesh_id_t::reference].max_level(), mpi::maximum()); + + std::vector> to_send(size_t(world.size())); + std::vector to_recv; + + std::vector req; + req.reserve(world.size()); #else auto min_level = mesh[mesh_id_t::reference].min_level(); auto max_level = mesh[mesh_id_t::reference].max_level(); -#endif - +#endif // SAMURAI_WITH_MPI for (std::size_t level = max_level; level > min_level; --level) { - update_ghost_subdomains(level, field, other_fields...); - update_ghost_periodic(level, field, other_fields...); +#ifdef SAMURAI_WITH_MPI + // first we set field[cell \ reference] to 0 + auto set_ghosts = difference(mesh[mesh_id_t::reference][level], mesh[mesh_id_t::cells][level]); + set_ghosts( + [&](const auto& interval_x, const auto& index_yz) + { + field(level, interval_x, index_yz).fill(0); + (other_fields(level, interval_x, index_yz).fill(0), ...); + }); +#endif // SAMURAI_WITH_MPI + //~ update_ghost_subdomains(level, field, other_fields...); + //~ update_ghost_periodic(level, field, other_fields...); auto set_at_levelm1 = intersection(mesh[mesh_id_t::reference][level], mesh[mesh_id_t::proj_cells][level - 1]).on(level - 1); set_at_levelm1.apply_op(variadic_projection(field, other_fields...)); } +#ifdef SAMURAI_WITH_MPI + const auto construct_set = [&](const auto& neighbor_mesh, const size_t level) -> auto + { + const auto set_my_ghosts_level_m1 = difference(mesh[mesh_id_t::reference][level - 1], mesh[mesh_id_t::cells][level - 1]); + const auto set_my_ghosts_level_m2 = difference(mesh[mesh_id_t::reference][level - 2], mesh[mesh_id_t::cells][level - 2]); + + const auto set_neighbor_ghosts_level_m1 = difference(neighbor_mesh[mesh_id_t::reference][level - 1], + neighbor_mesh[mesh_id_t::cells][level - 1]); + const auto set_neighbor_ghosts_level_m2 = difference(neighbor_mesh[mesh_id_t::reference][level - 2], + neighbor_mesh[mesh_id_t::cells][level - 2]); + const auto set_commun_ghosts_level_m1 = intersection(set_my_ghosts_level_m1, set_neighbor_ghosts_level_m1); + const auto set_commun_ghosts_level_m2 = intersection(set_my_ghosts_level_m2, set_neighbor_ghosts_level_m2); + + return intersection(mesh[mesh_id_t::cells][level], union_(set_commun_ghosts_level_m1, set_commun_ghosts_level_m2)).on(level); + }; + + size_t mpi_neighbor_id = 0; + req.clear(); + for (const auto& mpi_neighbor : mesh.mpi_neighbourhood()) + { + to_send[mpi_neighbor_id].clear(); + for (std::size_t level = max_level; level > min_level; --level) + { + auto set_to_reduce = construct_set(mpi_neighbor.mesh, level); + set_to_reduce( + [&](const auto& interval_x, const auto& index_yz) + { + std::copy(field(level, interval_x, index_yz).cbegin(), + field(level, interval_x, index_yz).cend(), + std::back_inserter(to_send[mpi_neighbor_id])); + if (world.rank() == 1) + { + std::cout << "send field = " << field(level, interval_x, index_yz) << std::endl; + } + }); + } + req.push_back(world.isend(mpi_neighbor.rank, mpi_neighbor.rank, to_send[mpi_neighbor_id++])); + } + for (const auto& mpi_neighbor : mesh.mpi_neighbourhood()) + { + world.recv(mpi_neighbor.rank, world.rank(), to_recv); + auto src_it = to_recv.cbegin(); + for (std::size_t level = max_level; level > min_level; --level) + { + auto set_to_reduce = construct_set(mpi_neighbor.mesh, level); + set_to_reduce( + [&](const auto& interval_x, const auto& index_yz) + { + //~ auto& dst_field = field(level, interval_x, index_yz); + for (auto dst_it = field(level, interval_x, index_yz).begin(); dst_it != field(level, interval_x, index_yz).end(); + ++dst_it, ++src_it) + { + if (world.rank() == 0) + { + std::cout << "recieve field value = " << *src_it << std::endl; + } + *dst_it += *src_it; + } + }); + } + } + mpi::wait_all(req.begin(), req.end()); +#endif // SAMURAI_WITH_MPI if (min_level > 0 && min_level != max_level) { update_bc(min_level - 1, field, other_fields...); From 20a58a57cd99b3c7936ebde98c14c727e963f3e5 Mon Sep 17 00:00:00 2001 From: Alexandre Hoffmann Date: Mon, 2 Jun 2025 09:18:21 +0200 Subject: [PATCH 2/3] -*-* --- include/samurai/algorithm/update.hpp | 21 ++++++--------------- include/samurai/mr/mesh.hpp | 1 + 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/include/samurai/algorithm/update.hpp b/include/samurai/algorithm/update.hpp index 4b7acfdec..9a626bfd7 100644 --- a/include/samurai/algorithm/update.hpp +++ b/include/samurai/algorithm/update.hpp @@ -136,18 +136,10 @@ namespace samurai #ifdef SAMURAI_WITH_MPI const auto construct_set = [&](const auto& neighbor_mesh, const size_t level) -> auto { - const auto set_my_ghosts_level_m1 = difference(mesh[mesh_id_t::reference][level - 1], mesh[mesh_id_t::cells][level - 1]); - const auto set_my_ghosts_level_m2 = difference(mesh[mesh_id_t::reference][level - 2], mesh[mesh_id_t::cells][level - 2]); - - const auto set_neighbor_ghosts_level_m1 = difference(neighbor_mesh[mesh_id_t::reference][level - 1], - neighbor_mesh[mesh_id_t::cells][level - 1]); - const auto set_neighbor_ghosts_level_m2 = difference(neighbor_mesh[mesh_id_t::reference][level - 2], - neighbor_mesh[mesh_id_t::cells][level - 2]); - - const auto set_commun_ghosts_level_m1 = intersection(set_my_ghosts_level_m1, set_neighbor_ghosts_level_m1); - const auto set_commun_ghosts_level_m2 = intersection(set_my_ghosts_level_m2, set_neighbor_ghosts_level_m2); - - return intersection(mesh[mesh_id_t::cells][level], union_(set_commun_ghosts_level_m1, set_commun_ghosts_level_m2)).on(level); + return intersection(mesh[mesh_id_t::reference][level], + mesh[mesh_id_t::proj_cells][level - 1], + neighbor_mesh[mesh_id_t::proj_cells][level - 1]) + .on(level - 1); }; size_t mpi_neighbor_id = 0; @@ -182,9 +174,8 @@ namespace samurai set_to_reduce( [&](const auto& interval_x, const auto& index_yz) { - //~ auto& dst_field = field(level, interval_x, index_yz); - for (auto dst_it = field(level, interval_x, index_yz).begin(); dst_it != field(level, interval_x, index_yz).end(); - ++dst_it, ++src_it) + auto dst_field = field(level, interval_x, index_yz); + for (auto dst_it = dst_field.begin(); dst_it != dst_field.end(); ++dst_it, ++src_it) { if (world.rank() == 0) { diff --git a/include/samurai/mr/mesh.hpp b/include/samurai/mr/mesh.hpp index 59f37bc55..5c1df6f00 100644 --- a/include/samurai/mr/mesh.hpp +++ b/include/samurai/mr/mesh.hpp @@ -412,6 +412,7 @@ namespace samurai this->update_neighbour_subdomain(); this->update_meshid_neighbour(mesh_id_t::cells_and_ghosts); this->update_meshid_neighbour(mesh_id_t::reference); + this->update_meshid_neighbour(mesh_id_t::proj_cells); } } From b08b6d64e5c6abc78d3ea5dce487292633851cbc Mon Sep 17 00:00:00 2001 From: Alexandre Hoffmann Date: Mon, 2 Jun 2025 17:06:40 +0200 Subject: [PATCH 3/3] update ghost works --- demos/FiniteVolume/advection_2d.cpp | 1 - include/samurai/algorithm/update.hpp | 23 ++++++++++------------- include/samurai/mr/adapt.hpp | 3 ++- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index c07d2296b..5d1bfefc4 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -247,7 +247,6 @@ int main(int argc, char* argv[]) auto MRadaptation = samurai::make_MRAdapt(u); MRadaptation(mr_epsilon, mr_regularity); save(path, filename, u, "_init"); - std::size_t nsave = 1; std::size_t nt = 0; diff --git a/include/samurai/algorithm/update.hpp b/include/samurai/algorithm/update.hpp index 9a626bfd7..3507e9196 100644 --- a/include/samurai/algorithm/update.hpp +++ b/include/samurai/algorithm/update.hpp @@ -115,10 +115,9 @@ namespace samurai auto min_level = mesh[mesh_id_t::reference].min_level(); auto max_level = mesh[mesh_id_t::reference].max_level(); #endif // SAMURAI_WITH_MPI + for (std::size_t level = max_level; level > min_level; --level) { -#ifdef SAMURAI_WITH_MPI - // first we set field[cell \ reference] to 0 auto set_ghosts = difference(mesh[mesh_id_t::reference][level], mesh[mesh_id_t::cells][level]); set_ghosts( [&](const auto& interval_x, const auto& index_yz) @@ -126,6 +125,12 @@ namespace samurai field(level, interval_x, index_yz).fill(0); (other_fields(level, interval_x, index_yz).fill(0), ...); }); + } + + for (std::size_t level = max_level; level > min_level; --level) + { +#ifdef SAMURAI_WITH_MPI + // first we set field[cell \ reference] to 0 #endif // SAMURAI_WITH_MPI //~ update_ghost_subdomains(level, field, other_fields...); //~ update_ghost_periodic(level, field, other_fields...); @@ -153,13 +158,9 @@ namespace samurai set_to_reduce( [&](const auto& interval_x, const auto& index_yz) { - std::copy(field(level, interval_x, index_yz).cbegin(), - field(level, interval_x, index_yz).cend(), + std::copy(field(level - 1, interval_x, index_yz).cbegin(), + field(level - 1, interval_x, index_yz).cend(), std::back_inserter(to_send[mpi_neighbor_id])); - if (world.rank() == 1) - { - std::cout << "send field = " << field(level, interval_x, index_yz) << std::endl; - } }); } req.push_back(world.isend(mpi_neighbor.rank, mpi_neighbor.rank, to_send[mpi_neighbor_id++])); @@ -174,13 +175,9 @@ namespace samurai set_to_reduce( [&](const auto& interval_x, const auto& index_yz) { - auto dst_field = field(level, interval_x, index_yz); + auto dst_field = field(level - 1, interval_x, index_yz); for (auto dst_it = dst_field.begin(); dst_it != dst_field.end(); ++dst_it, ++src_it) { - if (world.rank() == 0) - { - std::cout << "recieve field value = " << *src_it << std::endl; - } *dst_it += *src_it; } }); diff --git a/include/samurai/mr/adapt.hpp b/include/samurai/mr/adapt.hpp index cfd95a472..fbd0def2d 100644 --- a/include/samurai/mr/adapt.hpp +++ b/include/samurai/mr/adapt.hpp @@ -245,10 +245,11 @@ namespace samurai { update_tag_subdomains(level, m_tag, true); } - + //~ save(std::filesystem::current_path(), "ite_" + std::to_string(ite) + "_before_update_ghosts", {true, true}, mesh, m_fields); times::timers.stop("mesh adaptation"); update_ghost_mr(m_fields); times::timers.start("mesh adaptation"); + //~ save(std::filesystem::current_path(), "ite_" + std::to_string(ite) + "_after_update_ghosts", {true, true}, mesh, m_fields); for (std::size_t level = ((min_level > 0) ? min_level - 1 : 0); level < max_level - ite; ++level) {