diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index a739d79bd..6294cc8cc 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -15,6 +15,7 @@ jobs: - { name: GNU, CC: gcc-10, CXX: g++-10 } - { name: LLVM, CC: clang, CXX: clang++ } build_type: [Debug, Release] + no_mpi: [OFF, ON] steps: - uses: actions/checkout@v4 @@ -30,7 +31,7 @@ jobs: env: MPICH_CXX: ${{matrix.compiler.CXX}} MPICH_CC: ${{matrix.compiler.CC}} - run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install + run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=${{matrix.no_mpi}} - name: Build env: @@ -54,3 +55,20 @@ jobs: run: | cmake -S ${{github.workspace}}/doc -B ${{github.workspace}}/buildExample -DCMAKE_CXX_COMPILER=mpicxx -DSCOREC_PREFIX=${{github.workspace}}/build/install cmake --build ${{github.workspace}}/buildExample + + - name: Build MPI-NoMPI Example + # Test if a SCOREC_NO_MPI build works with MPI applications. + if: >- + matrix.compiler.name == 'GNU' && matrix.build_type == 'Release' && + matrix.no_mpi == 'ON' + env: + MPICH_CXX: ${{matrix.compiler.CXX}} + MPICH_CC: ${{matrix.compiler.CC}} + run: > + cmake -S ${{github.workspace}}/example/mpi-nompi + -B ${{github.workspace}}/example/mpi-nompi/build + -DCMAKE_CXX_COMPILER=mpicxx + -DSCOREC_PREFIX=${{github.workspace}}/build/install ; + cmake --build ${{github.workspace}}/example/mpi-nompi/build ; + ctest --test-dir ${{github.workspace}}/example/mpi-nompi/build + --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index c0a59fa36..a8af52867 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ endif() # This is the top level CMake file for the SCOREC build cmake_minimum_required(VERSION 3.8) -project(SCOREC VERSION 3.0.3 LANGUAGES CXX C) +project(SCOREC VERSION 4.0.0 LANGUAGES CXX C) include(cmake/bob.cmake) include(cmake/xsdk.cmake) @@ -29,6 +29,15 @@ else() bob_set_cxx_standard(14) endif() +# +# option related to MPI +# +option(SCOREC_NO_MPI "Disable MPI support. When enabled only serial execution is supported." OFF) +message(STATUS "SCOREC_NO_MPI: ${SCOREC_NO_MPI}") +if(SCOREC_NO_MPI AND ENABLE_CGNS) + message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with CGNS.") +endif() + # Set some default compiler flags that should always be used if(NOT USE_XSDK_DEFAULTS) bob_set_shared_libs() @@ -133,7 +142,10 @@ endif() macro(scorec_export_library target) bob_export_target(${target}) -install(FILES ${HEADERS} DESTINATION include) +install(FILES ${HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +target_include_directories(${target} INTERFACE + "$" +) endmacro(scorec_export_library) if(ENABLE_SIMMETRIX) @@ -162,6 +174,10 @@ if(ENABLE_CGNS) add_definitions(-DHAVE_CGNS) endif() +configure_file(SCOREC_config.h.in SCOREC_config.h) +install(FILES "${CMAKE_BINARY_DIR}/SCOREC_config.h" DESTINATION include) +include_directories(PUBLIC "$") + # Include the SCOREC project packages add_subdirectory(lion) add_subdirectory(pcu) diff --git a/SCOREC_config.h.in b/SCOREC_config.h.in new file mode 100644 index 000000000..7fe11839b --- /dev/null +++ b/SCOREC_config.h.in @@ -0,0 +1,7 @@ +#ifndef SCOREC_CONFIG_H +#define SCOREC_CONFIG_H + +#cmakedefine SCOREC_NO_MPI + +#endif // SCOREC_CONFIG_H + diff --git a/apf/CMakeLists.txt b/apf/CMakeLists.txt index 2c65af0ad..56109cc1a 100644 --- a/apf/CMakeLists.txt +++ b/apf/CMakeLists.txt @@ -93,7 +93,6 @@ add_library(apf ${SOURCES}) # Include directories target_include_directories(apf PUBLIC $ - $ ) # Link this library to these others diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 4bd3499b0..da01cd99b 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -545,8 +545,9 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe cgp_error_exit(); std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); - MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1, - MPI_INT, m->getPCU()->GetMPIComm()); + m->getPCU()->Allgather( + &numbersByElementType[o], allNumbersForThisType.data(), 1 + ); cgsize_t num = 0; for (int i = 0; i < m->getPCU()->Self(); i++) @@ -656,8 +657,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, } std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); - MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1, - MPI_INT, m->getPCU()->GetMPIComm()); + m->getPCU()->Allgather(&number, allNumbersForThisType.data(), 1); cgsize_t num = 0; for (int i = 0; i < m->getPCU()->Self(); i++) @@ -684,19 +684,16 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, } } std::vector cacheStarts(m->getPCU()->Peers(), 0); - MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1, - MPI_INT, m->getPCU()->GetMPIComm()); + m->getPCU()->Allgather(&cacheStart, cacheStarts.data(), 1); std::vector cacheEnds(m->getPCU()->Peers(), 0); - MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1, - MPI_INT, m->getPCU()->GetMPIComm()); + m->getPCU()->Allgather(&cacheEnd, cacheEnds.data(), 1); return std::make_pair(cacheStarts, cacheEnds); }; const auto globalElementList = [&m](const std::vector &bcList, std::vector &allElements) { std::vector sizes(m->getPCU()->Peers(), 0); // important initialiser const int l = bcList.size(); - MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1, - MPI_INT, m->getPCU()->GetMPIComm()); + m->getPCU()->Allgather(&l, sizes.data(), 1); int totalLength = 0; for (const auto &i : sizes) @@ -708,9 +705,15 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, displacement[i] = displacement[i - 1] + sizes[i - 1]; allElements.resize(totalLength); + #ifndef SCOREC_NO_MPI + PCU_Comm comm; + m->getPCU()->DupComm(&comm); MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(), - sizes.data(), displacement.data(), MPI_INT, - m->getPCU()->GetMPIComm()); + sizes.data(), displacement.data(), MPI_INT, comm); + MPI_Comm_free(&comm); + #else + std::copy(bcList.begin(), bcList.end(), allElements.begin()); + #endif }; const auto doVertexBC = [&](const auto &iter) { @@ -1046,7 +1049,8 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap sizes[2] = 0; // nodes are unsorted, as defined by api // Copy communicator - auto communicator = m->getPCU()->GetMPIComm(); + PCU_Comm communicator; + m->getPCU()->DupComm(&communicator); cgp_mpi_comm(communicator); // cgp_pio_mode(CGP_INDEPENDENT); @@ -1134,6 +1138,9 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap destroyGlobalNumbering(gcn); // cgp_close(cgns.index); + #ifndef SCOREC_NO_MPI + MPI_Comm_free(&communicator); + #endif } } // namespace diff --git a/apf_cap/CMakeLists.txt b/apf_cap/CMakeLists.txt index 597d6406a..654e89db2 100644 --- a/apf_cap/CMakeLists.txt +++ b/apf_cap/CMakeLists.txt @@ -39,7 +39,6 @@ include(GNUInstallDirs) target_include_directories(apf_cap PUBLIC $ - $ ) scorec_export_library(apf_cap) diff --git a/apf_sim/CMakeLists.txt b/apf_sim/CMakeLists.txt index a2dd93101..7606ef720 100644 --- a/apf_sim/CMakeLists.txt +++ b/apf_sim/CMakeLists.txt @@ -19,7 +19,6 @@ target_link_libraries(apf_sim PUBLIC ${SIMMODSUITE_LIBS}) target_include_directories(gmi_sim PUBLIC ${SIMMODSUITE_INCLUDE_DIR}) target_include_directories(apf_sim PUBLIC $ - $ ) option(ENABLE_FIELDSIM "Enable use of FieldSim from Simmetrix SimModSuite" FALSE) diff --git a/can/CMakeLists.txt b/can/CMakeLists.txt index ac0c95c91..c2a57a56d 100644 --- a/can/CMakeLists.txt +++ b/can/CMakeLists.txt @@ -11,7 +11,6 @@ add_library(can INTERFACE) # Include directories target_include_directories(can INTERFACE $ - $ ) scorec_export_library(can) diff --git a/crv/CMakeLists.txt b/crv/CMakeLists.txt index 992589d23..bcede1404 100644 --- a/crv/CMakeLists.txt +++ b/crv/CMakeLists.txt @@ -33,7 +33,6 @@ add_library(crv ${SOURCES}) # Include directories target_include_directories(crv INTERFACE $ - $ ) # Link this package to these libraries diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 07f52b7b0..0bcddce3a 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -16,13 +16,13 @@ set(SCOREC_PREFIX "" CACHE STRING "Directory where SCOREC is installed") # If SCOREC_PREFIX was specified, only link to that directory, # i.e. don't link to another installation in /usr/lib by mistake if (SCOREC_PREFIX) - find_package(SCOREC 3.0.0 REQUIRED CONFIG PATHS ${SCOREC_PREFIX} NO_DEFAULT_PATH) + find_package(SCOREC 4 REQUIRED CONFIG PATHS ${SCOREC_PREFIX} NO_DEFAULT_PATH) else() # IF SCOREC_PREFIX was not specified, look in typical system directories, # and also in CMAKE_PREFIX_PATH (environment variable) find_package( SCOREC #package name, has to be SCOREC - 3.0.0 #version. can be omitted, and will match any installed version + 4 #version. can be omitted, and will match any installed version #greater than or equal to this one, as long as the major number #is the same REQUIRED #indicate that SCOREC is really needed to compile diff --git a/doc/myprogram.cpp b/doc/myprogram.cpp index 4b8ad85cf..59df0f823 100644 --- a/doc/myprogram.cpp +++ b/doc/myprogram.cpp @@ -1,9 +1,13 @@ #include #include "mylibrary.h" int main(int argc, char** argv) { +#ifndef SCOREC_NO_MPI MPI_Init(&argc,&argv); - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); +#endif + pcu::PCU PCUObj; makeMesh(&PCUObj); +#ifndef SCOREC_NO_MPI MPI_Finalize(); +#endif return 0; } diff --git a/dsp/CMakeLists.txt b/dsp/CMakeLists.txt index 2851c08fe..62ec50144 100644 --- a/dsp/CMakeLists.txt +++ b/dsp/CMakeLists.txt @@ -28,7 +28,6 @@ add_library(dsp ${SOURCES}) # Include directories target_include_directories(apf INTERFACE $ - $ ) # Link this package to these libraries diff --git a/example/mpi-nompi/CMakeLists.txt b/example/mpi-nompi/CMakeLists.txt new file mode 100644 index 000000000..eaea8d8ba --- /dev/null +++ b/example/mpi-nompi/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.8) +project(mpi-nompi VERSION 1.0.0 LANGUAGES CXX) + +# Allow the user to indicate where they installed SCOREC +# via "-DSCOREC_PREFIX=/home/somewhere" when calling `cmake` +set(SCOREC_PREFIX "" CACHE STRING "Directory where SCOREC is installed") + +# If SCOREC_PREFIX was specified then use only that directory. +if (SCOREC_PREFIX) + find_package(SCOREC 4 REQUIRED CONFIG PATHS "${SCOREC_PREFIX}" NO_DEFAULT_PATH) +else() + find_package(SCOREC 4 REQUIRED CONFIG) +endif() + +add_executable(hello hello.cc) +target_link_libraries(hello PRIVATE SCOREC::core) +add_executable(test_coll test_coll.cc) +target_link_libraries(test_coll PRIVATE SCOREC::core) + +enable_testing() +add_test(NAME coll1 COMMAND mpirun -np 1 $) +add_test(NAME coll2 COMMAND mpirun -np 2 $) +add_test(NAME coll4 COMMAND mpirun -np 4 $) + diff --git a/example/mpi-nompi/README.md b/example/mpi-nompi/README.md new file mode 100644 index 000000000..582e4ea3e --- /dev/null +++ b/example/mpi-nompi/README.md @@ -0,0 +1,6 @@ +# Example: Using SCOREC_NO_MPI in an MPI application + +This example demonstrates how to combine a SCOREC_NO_MPI build of core with an +application that is using MPI. This is the prime use case for the SCOREC_NO_MPI +option. + diff --git a/example/mpi-nompi/hello.cc b/example/mpi-nompi/hello.cc new file mode 100644 index 000000000..7fa31ee5c --- /dev/null +++ b/example/mpi-nompi/hello.cc @@ -0,0 +1,89 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +template +void print_vector(const std::vector &v); + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + { // pcu object scope + pcu::PCU PCUObj; + // Print rank info. + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (rank == 0) + std::cout << "MPI size: " << size << "; PCU size: " << + PCUObj.Peers() << std::endl; + for (int i = 0; i < size; ++i) { + if (rank == i) + std::cout << "Hello from MPI rank: " << rank << "; PCU rank: " << + PCUObj.Self() << std::endl; + MPI_Barrier(MPI_COMM_WORLD); + } + // Test SCOREC functions. + gmi_register_mesh(); + apf::Mesh2* m = apf::makeMdsBox(1, 1, 1, 1, 1, 1, 0, &PCUObj); + apf::destroyMesh(m); + // MPI Allreduce + int val = rank + 1, sum; + MPI_Allreduce(&val, &sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + int pval = rank + 1, psum; + psum = PCUObj.Add(pval); + for (int i = 0; i < size; ++i) { + if (rank == i) + std::cout << "MPI(" << rank << ") sum: " << sum << + "; PCU sum: " << psum << std::endl; + MPI_Barrier(MPI_COMM_WORLD); + } + // Test Allgather + std::vector vals1(size); + MPI_Allgather(&val, 1, MPI_INT, vals1.data(), 1, MPI_INT, MPI_COMM_WORLD); + std::vector pvals1(PCUObj.Peers()); + PCUObj.Allgather(&val, pvals1.data(), 1); + for (int i = 0; i < size; ++i) { + if (rank == i) { + std::cout << "MPI(" << rank << ") gather: "; + print_vector(vals1); + std::cout << "; PCU gather: "; + print_vector(pvals1); + std::cout << std::endl; + } + MPI_Barrier(MPI_COMM_WORLD); + } + int val2[2] = {val, -val}; + std::vector vals2(2 * size); + MPI_Allgather(val2, 2, MPI_INT, vals2.data(), 2, MPI_INT, MPI_COMM_WORLD); + std::vector pvals2(2 * PCUObj.Peers()); + PCUObj.Allgather(val2, pvals2.data(), 2); + for (int i = 0; i < size; ++i) { + if (rank == i) { + std::cout << "MPI(" << rank << ") gather2: "; + print_vector(vals2); + std::cout << "; PCU gather2: "; + print_vector(pvals2); + std::cout << std::endl; + } + MPI_Barrier(MPI_COMM_WORLD); + } + } // pcu object scope + MPI_Finalize(); + return 0; +} + +template +void print_vector(const std::vector& v) { + std::cout << "["; + for (int i = 0; i < v.size(); ++i) { + if (i != 0) std::cout << ", "; + std::cout << v[i]; + } + std::cout << "]"; +} diff --git a/example/mpi-nompi/test_coll.cc b/example/mpi-nompi/test_coll.cc new file mode 100644 index 000000000..246b74d82 --- /dev/null +++ b/example/mpi-nompi/test_coll.cc @@ -0,0 +1,62 @@ +#include +#include + +#include +#include +#include + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + { // pcu object scope + pcu::PCU PCUObj; + // Print rank info. + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (rank == 0) + std::cout << "MPI size: " << size << "; PCU size: " << + PCUObj.Peers() << std::endl; + #ifdef SCOREC_NO_MPI + PCU_ALWAYS_ASSERT(PCUObj.Self() == 0); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 1); + #else + PCU_ALWAYS_ASSERT(PCUObj.Self() == rank); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == size); + #endif + // MPI Allreduce + int val = rank + 1, sum; + MPI_Allreduce(&val, &sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + int pval = PCUObj.Self() + 1, psum; + psum = PCUObj.Add(pval); + for (int i = 0; i < size; ++i) { + PCU_ALWAYS_ASSERT(sum == (size * (size + 1)) / 2); + PCU_ALWAYS_ASSERT(psum == (PCUObj.Peers() * (PCUObj.Peers() + 1)) / 2); + } + // Test Allgather + std::vector vals1(size); + MPI_Allgather(&val, 1, MPI_INT, vals1.data(), 1, MPI_INT, MPI_COMM_WORLD); + std::vector pvals1(PCUObj.Peers()); + PCUObj.Allgather(&pval, pvals1.data(), 1); + for (int i = 0; i < size; ++i) + PCU_ALWAYS_ASSERT(vals1[i] == i + 1); + for (int i = 0; i < PCUObj.Peers(); ++i) + PCU_ALWAYS_ASSERT(pvals1[i] == i + 1); + int val2[2] = {val, -val}; + std::vector vals2(2 * size); + MPI_Allgather(val2, 2, MPI_INT, vals2.data(), 2, MPI_INT, MPI_COMM_WORLD); + int pval2[2] = {pval, -pval}; + std::vector pvals2(2 * PCUObj.Peers()); + PCUObj.Allgather(pval2, pvals2.data(), 2); + for (int i = 0; i < size; ++i) { + PCU_ALWAYS_ASSERT(vals2[i * 2] == i + 1); + PCU_ALWAYS_ASSERT(vals2[i * 2 + 1] == -(i + 1)); + } + for (int i = 0; i < PCUObj.Peers(); ++i) { + PCU_ALWAYS_ASSERT(pvals2[i * 2] == i + 1); + PCU_ALWAYS_ASSERT(pvals2[i * 2 + 1] == -(i + 1)); + } + } // pcu object scope + MPI_Finalize(); + return 0; +} + diff --git a/gmi/CMakeLists.txt b/gmi/CMakeLists.txt index b132ad600..c23421cf1 100644 --- a/gmi/CMakeLists.txt +++ b/gmi/CMakeLists.txt @@ -31,7 +31,6 @@ add_library(gmi ${SOURCES}) target_link_libraries(gmi pcu lion) target_include_directories(gmi INTERFACE $ - $ ) scorec_export_library(gmi) diff --git a/gmi_cap/CMakeLists.txt b/gmi_cap/CMakeLists.txt index 5e2cb3ea2..3fa431f34 100644 --- a/gmi_cap/CMakeLists.txt +++ b/gmi_cap/CMakeLists.txt @@ -18,7 +18,6 @@ target_link_libraries(gmi_cap PUBLIC gmi pcu lion capstone_module framework_test # Include directories target_include_directories(gmi_cap PUBLIC $ - $ ) # make sure the compiler can find the config header target_include_directories(gmi_cap PRIVATE diff --git a/gmi_sim/CMakeLists.txt b/gmi_sim/CMakeLists.txt index dda337a5b..17df8ba74 100644 --- a/gmi_sim/CMakeLists.txt +++ b/gmi_sim/CMakeLists.txt @@ -26,7 +26,6 @@ target_link_libraries(gmi_sim PUBLIC gmi pcu ${SIMMODSUITE_LIBS}) # Include directories target_include_directories(gmi_sim PUBLIC $ - $ ) # make sure the compiler can find the config header target_include_directories(gmi_sim PRIVATE diff --git a/lion/CMakeLists.txt b/lion/CMakeLists.txt index 1a46f76a0..84f510b1a 100644 --- a/lion/CMakeLists.txt +++ b/lion/CMakeLists.txt @@ -33,7 +33,6 @@ add_library(lion ${SOURCES}) # Include directories target_include_directories(lion INTERFACE $ - $ ) # Do extra work if compression is enabled diff --git a/ma/CMakeLists.txt b/ma/CMakeLists.txt index 59c0abb21..42fb3cd1a 100644 --- a/ma/CMakeLists.txt +++ b/ma/CMakeLists.txt @@ -72,7 +72,6 @@ add_library(ma ${SOURCES}) # Include directories target_include_directories(ma INTERFACE $ - $ ) # Link this library to these libraries diff --git a/mds/CMakeLists.txt b/mds/CMakeLists.txt index 528c92525..a2d9f8197 100644 --- a/mds/CMakeLists.txt +++ b/mds/CMakeLists.txt @@ -47,7 +47,6 @@ add_library(mds ${SOURCES}) # Include directories target_include_directories(mds INTERFACE $ - $ ) # for the generated mds_config.h # private because no one outside MDS needs it diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index fd811f914..54f483607 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -1054,7 +1054,8 @@ apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap static_assert(std::is_same::value, "cgsize_t not compiled as int"); int cgid = -1; - auto comm = PCU_Get_Comm(h); + PCU_Comm comm; + PCU_Comm_Dup(h, &comm); cgp_mpi_comm(comm); cgp_pio_mode(CGP_INDEPENDENT); cgp_open(fname.c_str(), CG_MODE_READ, &cgid); @@ -1636,6 +1637,9 @@ apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap cgp_close(cgid); else cg_close(cgid); +#ifndef SCOREC_NO_MPI + MPI_Comm_free(&comm); +#endif { apf::MeshTag *tag = nullptr; diff --git a/mth/CMakeLists.txt b/mth/CMakeLists.txt index 1867870a1..e51d2df8d 100644 --- a/mth/CMakeLists.txt +++ b/mth/CMakeLists.txt @@ -20,7 +20,6 @@ add_library(mth ${SOURCES}) # Include directories target_include_directories(mth INTERFACE $ - $ ) # Link this library to these libraries diff --git a/omega_h/CMakeLists.txt b/omega_h/CMakeLists.txt index 750fa0538..210994820 100644 --- a/omega_h/CMakeLists.txt +++ b/omega_h/CMakeLists.txt @@ -26,7 +26,6 @@ endif() # Include directories target_include_directories(apf_omega_h INTERFACE $ - $ ) # Link this package to these libraries diff --git a/parma/CMakeLists.txt b/parma/CMakeLists.txt index 715a2a248..1d859d855 100644 --- a/parma/CMakeLists.txt +++ b/parma/CMakeLists.txt @@ -70,7 +70,6 @@ add_library(parma ${SOURCES}) target_include_directories(parma INTERFACE $ - $ PRIVATE $ $ diff --git a/parma/diffMC/maximalIndependentSet/mis.h b/parma/diffMC/maximalIndependentSet/mis.h index 291d7c6a4..98f5c1ef5 100644 --- a/parma/diffMC/maximalIndependentSet/mis.h +++ b/parma/diffMC/maximalIndependentSet/mis.h @@ -7,8 +7,6 @@ #include #include -#include "mpi.h" - #include #include "PCU.h" diff --git a/parma/diffMC/maximalIndependentSet/misLuby.cc b/parma/diffMC/maximalIndependentSet/misLuby.cc index ecabc915a..34195af14 100644 --- a/parma/diffMC/maximalIndependentSet/misLuby.cc +++ b/parma/diffMC/maximalIndependentSet/misLuby.cc @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index 799428082..4291bcd70 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -48,17 +48,13 @@ static void runInGroups( int groupRank = inMap(self); int group = groupMap(self); - MPI_Comm groupComm; - MPI_Comm_split(expandedPCU->GetMPIComm(), group, groupRank, &groupComm); - auto groupedPCU = std::unique_ptr(new pcu::PCU(groupComm)); + auto groupedPCU = expandedPCU->Split(group, groupRank); if (m){ m->switchPCU(groupedPCU.get()); apf::remapPartition(m, inMap); } code.PCUObj = std::move(groupedPCU); code.run(group); - - MPI_Comm_free(&groupComm); if (m){ m->switchPCU(expandedPCU); apf::remapPartition(m, outMap); diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 51ccbcdea..79202f2a2 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -18,12 +18,17 @@ set(SOURCES pcu_mpi.c pcu_msg.c pcu_order.c - pcu_pmpi.c pcu_util.c noto/noto_malloc.c reel/reel.c PCU.cc) +if(SCOREC_NO_MPI) + set(SOURCES ${SOURCES} pcu_pnompi.c) +else() + set(SOURCES ${SOURCES} pcu_pmpi.c) +endif() + # Package headers set(HEADERS PCU_C.h @@ -66,7 +71,6 @@ target_include_directories(pcu PUBLIC $ $ - $ PRIVATE $ ) diff --git a/pcu/PCU.cc b/pcu/PCU.cc index c0f90d300..062abb913 100644 --- a/pcu/PCU.cc +++ b/pcu/PCU.cc @@ -49,8 +49,27 @@ #include #include #include +#include namespace pcu { +void Init(int *argc, char ***argv) { +#ifndef SCOREC_NO_MPI + int flag; + MPI_Initialized(&flag); + if (!flag) MPI_Init(argc, argv); +#else + (void) argc, (void) argv; +#endif +} + +void Finalize() { +#ifndef SCOREC_NO_MPI + int flag; + MPI_Finalized(&flag); + if (!flag) MPI_Finalize(); +#endif +} + int PCU::Peers() const noexcept { return pcu_mpi_size(mpi_); } int PCU::Self() const noexcept { return pcu_mpi_rank(mpi_); } void PCU::Begin() noexcept { pcu_msg_start(mpi_, msg_); } @@ -125,6 +144,17 @@ void PCU::Order(bool on) { void PCU::Barrier() { pcu_barrier(mpi_, &(msg_->coll)); } int PCU::Or(int c) noexcept { return Max(c); } int PCU::And(int c) noexcept { return Min(c); } + +std::unique_ptr PCU::Split(int color, int key) noexcept { + PCU_Comm newcomm; + pcu_mpi_split(mpi_, color, key, &newcomm); + PCU* splitpcu = new PCU(newcomm); + return std::unique_ptr(splitpcu); +} +int PCU::DupComm(PCU_Comm* newcomm) const noexcept { + return pcu_mpi_dup(mpi_, newcomm); +} + int PCU::Packed(int to_rank, size_t *size) noexcept { if ((to_rank < 0) || (to_rank >= Peers())) reel_fail("Invalid rank in Comm_Packed"); @@ -193,7 +223,15 @@ void PCU::DebugOpen() noexcept { double GetMem() noexcept { return pcu_get_mem(); } void Protect() noexcept { reel_protect(); } -double Time() noexcept { return MPI_Wtime(); } +double Time() noexcept { +#ifndef SCOREC_NO_MPI + return MPI_Wtime(); +#else + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + return (double)now.tv_sec + now.tv_nsec * 1.0e-9; +#endif +} void PCU::DebugPrint(const char *format, ...) noexcept { va_list args; @@ -207,7 +245,14 @@ void PCU::DebugPrint(const char *format, va_list args) noexcept { vfprintf(msg_->file, format, args); fflush(msg_->file); } -PCU::PCU(MPI_Comm comm) { +PCU::PCU() : +#ifndef SCOREC_NO_MPI +PCU(MPI_COMM_WORLD) +#else +PCU(0) +#endif +{} +PCU::PCU(PCU_Comm comm) { mpi_ = new pcu_mpi_t; msg_ = new pcu_msg; pcu_mpi_init(comm, mpi_); @@ -220,6 +265,7 @@ PCU::PCU(MPI_Comm comm) { PCU::~PCU() noexcept { pcu_mpi_finalize(mpi_); delete mpi_; + if (msg_->order) pcu_order_free(msg_->order); pcu_free_msg(msg_); delete msg_; } @@ -232,23 +278,12 @@ PCU &PCU::operator=(PCU && other) noexcept { std::swap(msg_, other.msg_); return *this; } -MPI_Comm PCU::GetMPIComm() const noexcept { return mpi_->original_comm; } - -MPI_Comm PCU::SwitchMPIComm(MPI_Comm newcomm) noexcept { - if(newcomm == mpi_->original_comm) { - return mpi_->original_comm; - } - auto original_comm = mpi_->original_comm; - pcu_mpi_finalize(mpi_); - pcu_mpi_init(newcomm, mpi_); - return original_comm; -} /* template implementations */ template void PCU::Add(T *p, size_t n) noexcept { pcu_allreduce( mpi_, &(msg_->coll), - [](void *local, void *incoming, size_t size) { + [](int, int, void *local, void *incoming, size_t size) { auto *a = static_cast(local); auto *b = static_cast(incoming); size_t n = size / sizeof(T); @@ -264,7 +299,7 @@ template T PCU::Add(T p) noexcept { template void PCU::Min(T *p, size_t n) noexcept { pcu_allreduce( mpi_, &(msg_->coll), - [](void *local, void *incoming, size_t size) { + [](int, int, void *local, void *incoming, size_t size) { auto *a = static_cast(local); auto *b = static_cast(incoming); size_t n = size / sizeof(T); @@ -280,7 +315,7 @@ template T PCU::Min(T p) noexcept { template void PCU::Max(T *p, size_t n) noexcept { pcu_allreduce( mpi_, &(msg_->coll), - [](void *local, void *incoming, size_t size) { + [](int, int, void *local, void *incoming, size_t size) { auto *a = static_cast(local); auto *b = static_cast(incoming); size_t n = size / sizeof(T); @@ -299,7 +334,7 @@ template void PCU::Exscan(T *p, size_t n) noexcept { originals[i] = p[i]; pcu_scan( mpi_, &(msg_->coll), - [](void *local, void *incoming, size_t size) { + [](int, int, void *local, void *incoming, size_t size) { auto *a = static_cast(local); auto *b = static_cast(incoming); size_t n = size / sizeof(T); @@ -316,6 +351,12 @@ template T PCU::Exscan(T p) noexcept { Exscan(&p, 1); return p; } + +template +void PCU::Allgather(const T *send_data, T *recv_data, size_t n) noexcept { + pcu_allgather(mpi_, &(msg_->coll), send_data, recv_data, n * sizeof(T)); +} + #define PCU_EXPL_INST_DECL(T) \ template void PCU::Add(T * p, size_t n) noexcept; \ template T PCU::Add(T p) noexcept; \ @@ -324,7 +365,8 @@ template T PCU::Exscan(T p) noexcept { template void PCU::Max(T * p, size_t n) noexcept; \ template T PCU::Max(T p) noexcept; \ template void PCU::Exscan(T * p, size_t n) noexcept; \ - template T PCU::Exscan(T p) noexcept; + template T PCU::Exscan(T p) noexcept; \ + template void PCU::Allgather(const T *in, T *out, size_t n) noexcept; PCU_EXPL_INST_DECL(int) PCU_EXPL_INST_DECL(size_t) PCU_EXPL_INST_DECL(long) diff --git a/pcu/PCU.h b/pcu/PCU.h index dd56b0138..7b588758c 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -1,9 +1,9 @@ #ifndef SCOREC_PCU_H #define SCOREC_PCU_H +#include #include #include //va_list -#include #include "pcu_defines.h" struct pcu_msg_struct; @@ -12,7 +12,8 @@ struct pcu_mpi_struct; namespace pcu { class PCU { public: - explicit PCU(MPI_Comm comm); + PCU(); + explicit PCU(PCU_Comm comm); ~PCU() noexcept; PCU(PCU const &) = delete; PCU(PCU &&) noexcept; @@ -26,7 +27,6 @@ class PCU { * @return The number of ranks in the communicator. */ [[nodiscard]] int Peers() const noexcept; - [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; [[nodiscard]] PCU_t GetCHandle() {PCU_t h; h.ptr=this; return h;} /*recommended message passing API*/ @@ -69,11 +69,41 @@ class PCU { template [[nodiscard]] T Max(T p) noexcept; template void Exscan(T *p, size_t n) noexcept; template [[nodiscard]] T Exscan(T p) noexcept; + template + void Allgather(const T *send, T *recv, size_t n) noexcept; /*bitwise operations*/ [[nodiscard]] int Or(int c) noexcept; [[nodiscard]] int And(int c) noexcept; + /** + * @brief Split a communicator into distinct subgroups. + * + * The resulting communicator is marked owned and automatically free the + * underlying communicator. This can be disabled with PCU::OwnsComm(bool). In + * that case, the user is responsible for cleanup. + * + * @param color subgroup indicator. + * @param key used for subgroup ordering; specify 0 if you don't care. + * @return a new communicator defined on the resulting subgroup. + */ + std::unique_ptr Split(int color, int key) noexcept; + + /** + * @brief Duplicate the underlying communicator. + * + * It is the user's responsiblity to cleanup the returned communicator. This + * function may be used to initialize other libraries using the PCU-defined + * communication group. + * + * If SCOREC::core was compiled with the SCOREC_NO_MPI flag, the return value + * is not meaningful. + * + * @param[out] newcomm The output address for the new communicator copy. + * @return 0 on success. + */ + int DupComm(PCU_Comm* newcomm) const noexcept; + /*lesser-used APIs*/ int Packed(int to_rank, size_t *size) noexcept; int From(int *from_rank) noexcept; @@ -85,17 +115,6 @@ class PCU { /* Debug functions */ void DebugOpen() noexcept; - MPI_Comm SwitchMPIComm(MPI_Comm) noexcept; - - //struct MPIComms { - // MPI_Comm original; - // MPI_Comm user; - // MPI_Comm coll; - //}; - // takes ownership of newcomms.user & newcomms.coll - // user responsibility to free returned user/coll comm - //MPIComms SwitchMPIComms(MPIComms& newcomms) noexcept; - private: pcu_msg_struct *msg_; pcu_mpi_struct *mpi_; @@ -107,7 +126,20 @@ void Protect() noexcept; /*MPI_Wtime() equivalent*/ [[nodiscard]] double Time() noexcept; -PCU* PCU_GetGlobal(); +/** + * @brief Initialize the underlying parallel library. + * + * This may be MPI (or a stub, given SCOREC_NO_MPI). This function abstracts + * the difference. + */ +void Init(int *argc, char ***argv); +/** + * @brief Finalize the underlying parallel library. + * + * This may be MPI (or a stub, given SCOREC_NO_MPI). This function abstracts + * the difference. + */ +void Finalize(); /* explicit instantiations of template functions */ #define PCU_EXPL_INST_DECL(T) \ @@ -118,7 +150,9 @@ PCU* PCU_GetGlobal(); extern template void PCU::Max(T * p, size_t n) noexcept; \ extern template T PCU::Max(T p) noexcept; \ extern template void PCU::Exscan(T * p, size_t n) noexcept; \ - extern template T PCU::Exscan(T p) noexcept; + extern template T PCU::Exscan(T p) noexcept; \ + extern template \ + void PCU::Allgather(const T *send, T *recv, size_t n) noexcept; PCU_EXPL_INST_DECL(int) PCU_EXPL_INST_DECL(size_t) PCU_EXPL_INST_DECL(long) diff --git a/pcu/PCU_C.h b/pcu/PCU_C.h index e8a05169c..2c7889c5a 100644 --- a/pcu/PCU_C.h +++ b/pcu/PCU_C.h @@ -1,7 +1,6 @@ #ifndef PCU_C_H #define PCU_C_H #include "pcu_defines.h" -#include #ifdef __cplusplus #include @@ -15,6 +14,9 @@ extern "C" { typedef struct PCU_t PCU_t; +void PCU_Init(int *argc, char ***argv); +void PCU_Finalize(void); + int PCU_Comm_Init(PCU_t* h); int PCU_Comm_Free(PCU_t* h); @@ -22,6 +24,9 @@ int PCU_Comm_Free(PCU_t* h); int PCU_Comm_Self(PCU_t h); int PCU_Comm_Peers(PCU_t h); +int PCU_Comm_Dup(PCU_t h, PCU_Comm* newcomm); +void PCU_Comm_Split(PCU_t h, int color, int key, PCU_t* newpcu); + /*recommended message passing API*/ void PCU_Comm_Begin(PCU_t h); int PCU_Comm_Pack(PCU_t h, int to_rank, const void* data, size_t size); @@ -96,10 +101,6 @@ int PCU_Comm_Size(PCU_t h, int* size); /*deprecated method enum*/ -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm(PCU_t h); - /*stack trace helpers using GNU/Linux*/ void PCU_Protect(void); @@ -109,12 +110,8 @@ double PCU_Time(void); /*Memory usage*/ double PCU_GetMem(void); -/*Access global variable*/ -PCU_t PCU_Get_Global_Handle(void); - - #ifdef __cplusplus } /* extern "C" */ #endif -#endif \ No newline at end of file +#endif diff --git a/pcu/pcu_c.cc b/pcu/pcu_c.cc index c763cb19f..ad5f91681 100644 --- a/pcu/pcu_c.cc +++ b/pcu/pcu_c.cc @@ -1,5 +1,6 @@ #include "PCU_C.h" #include "PCU.h" +#include "pcu_pmpi.h" #include "reel.h" #include @@ -7,10 +8,18 @@ extern "C" { +void PCU_Init(int *argc, char ***argv) { + pcu::Init(argc, argv); +} + +void PCU_Finalize(void) { + pcu::Finalize(); +} + int PCU_Comm_Init(PCU_t* h) { if (h->ptr != nullptr) reel_fail("nested calls to Comm_Init"); - pcu::PCU* pcu_object = new pcu::PCU(MPI_COMM_WORLD); + pcu::PCU* pcu_object = new pcu::PCU(); h->ptr = static_cast(pcu_object); return PCU_SUCCESS; } @@ -35,6 +44,23 @@ int PCU_Comm_Peers(PCU_t h) { return static_cast(h.ptr)->Peers(); } +int PCU_Comm_Dup(PCU_t h, PCU_Comm* newcomm) { + if (h.ptr == nullptr) + reel_fail("PCU_Comm_Dup called before PCU_Comm_Init"); + return static_cast(h.ptr)->DupComm(newcomm); +} + +void PCU_Comm_Split(PCU_t h, int color, int key, PCU_t* newpcu) { + if (h.ptr == nullptr) + reel_fail("PCU_Comm_Split called before PCU_Comm_Init"); + if (newpcu == nullptr) + reel_fail("PCU_Comm_Split received NULL newpcu."); + if (newpcu->ptr != nullptr) + reel_fail("PCU_Comm_Split received an initialized newpcu."); + auto newpcu_ptr = static_cast(h.ptr)->Split(color, key); + newpcu->ptr = newpcu_ptr.release(); +} + void PCU_Comm_Begin(PCU_t h) { if (h.ptr == nullptr) reel_fail("Comm_Begin called before Comm_Init"); @@ -368,7 +394,6 @@ int PCU_Comm_Size(PCU_t h, int *size) { bool PCU_Comm_Initialized(PCU_t h) { return h.ptr != nullptr; } - /** \brief Returns in * \a size the number of bytes being sent to \a to_rank. \details Returns the size of the buffer being sent to \a to_rank. This function should be called after PCU_Comm_Start and before @@ -466,30 +491,6 @@ void *PCU_Comm_Extract(PCU_t h, size_t size) { return static_cast(h.ptr)->Extract(size); } -/** \brief Reinitializes PCU with a new MPI communicator. - \details All of PCU's logic is based off two duplicates - of this communicator, so you can safely get PCU to act - on sub-groups of processes using this function. - This call should be collective over all processes - in the previous communicator. This is a very heavy weight function - and should be used sparingly. - */ -void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm) { - if (h.ptr == nullptr) - reel_fail("Switch_Comm called before Comm_Init"); - static_cast(h.ptr)->SwitchMPIComm(new_comm); -} - -/** \brief Return the current MPI communicator - \details Returns the communicator given to the - most recent PCU_Switch_Comm call, or MPI_COMM_WORLD - otherwise. - */ -MPI_Comm PCU_Get_Comm(PCU_t h) { - if (h.ptr == nullptr) - reel_fail("Get_Comm called before Comm_Init"); - return static_cast(h.ptr)->GetMPIComm(); -} /** \brief Return the time in seconds since some time in the past */ @@ -500,4 +501,4 @@ void PCU_Protect(void) { return pcu::Protect(); } double PCU_GetMem(void) { return pcu::GetMem(); } -} \ No newline at end of file +} diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 01c89445f..f8c8da2f8 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -28,8 +28,10 @@ static int ceil_log2(int n) return r; } -void pcu_merge_assign(void* local, void* incoming, size_t size) +void pcu_merge_assign(int peers, int bit, void* local, void* incoming, + size_t size) { + (void) peers, (void) bit; memcpy(local,incoming,size); } @@ -64,7 +66,8 @@ static bool end_coll_step(pcu_mpi_t* mpi, pcu_coll* c) if (c->message.buffer.size != incoming.buffer.size) reel_fail("PCU unexpected incoming message.\n" "Most likely a PCU collective was not called by all ranks."); - c->merge(c->message.buffer.start,incoming.buffer.start,incoming.buffer.size); + c->merge(pcu_mpi_size(mpi), c->bit, c->message.buffer.start, + incoming.buffer.start, incoming.buffer.size); pcu_free_message(&incoming); return true; } @@ -373,6 +376,15 @@ static pcu_pattern scan_down = .shift = scan_down_shift, }; +static pcu_pattern gather = +{ + .begin_bit = reduce_begin_bit, + .end_bit = reduce_end_bit, + .action = reduce_action, + .peer = reduce_peer, + .shift = reduce_shift, +}; + void pcu_reduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { pcu_make_coll(mpi, c,&reduce,m); @@ -403,6 +415,30 @@ void pcu_scan(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size while(pcu_progress_coll(mpi, c)); } +void pcu_merge_gather(int peers, int bit, void *local, void *incoming, + size_t size) { + size_t block_size = size / peers; + // local has `bit` blocks. + // incoming may have `bit` (if peers is a power of 2) or `bit - 1` blocks. + // either way, writing `size - bit * block_size` prevents buffer overrun. + // Also, all incoming blocks are from greater ranks, so they go to the right. + memcpy(local + bit * block_size, incoming, size - bit * block_size); +} + +void pcu_gather(pcu_mpi_t* mpi, pcu_coll* c, const void *send_data, + void *recv_data, size_t size) { + memcpy(recv_data, send_data, size); + pcu_make_coll(mpi, c, &gather, pcu_merge_gather); + pcu_begin_coll(mpi, c, recv_data, size * pcu_mpi_size(mpi)); + while (pcu_progress_coll(mpi, c)); +} + +void pcu_allgather(pcu_mpi_t* mpi, pcu_coll* c, const void *send_data, + void *recv_data, size_t size) { + pcu_gather(mpi, c, send_data, recv_data, size); + pcu_bcast(mpi, c, recv_data, size * pcu_mpi_size(mpi)); +} + /* a barrier is just an allreduce of nothing in particular */ void pcu_begin_barrier(pcu_mpi_t* mpi, pcu_coll* c) { diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index f69bf7a08..2942cf787 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -29,8 +29,10 @@ extern "C" { arguments are usually arrays of some type, and the operations is sum, min, max, etc. */ -typedef void pcu_merge(void* local, void* incoming, size_t size); -void pcu_merge_assign(void* local, void* incoming, size_t size); +typedef void pcu_merge(int peers, int bit, void* local, void* incoming, + size_t size); +void pcu_merge_assign(int peers, int bit, void* local, void* incoming, + size_t size); /* Enumerated actions that a rank takes during one step of the communication pattern */ @@ -76,6 +78,8 @@ void pcu_reduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); void pcu_bcast(pcu_mpi_t*, pcu_coll* c, void* data, size_t size); void pcu_allreduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); void pcu_scan(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_allgather(pcu_mpi_t* mpi, pcu_coll* c, const void *send_data, + void *recv_data, size_t size); void pcu_begin_barrier(pcu_mpi_t*,pcu_coll* c); bool pcu_barrier_done(pcu_mpi_t*, pcu_coll* c); diff --git a/pcu/pcu_defines.h b/pcu/pcu_defines.h index acf8861e5..fbe478daf 100644 --- a/pcu/pcu_defines.h +++ b/pcu/pcu_defines.h @@ -1,6 +1,11 @@ #ifndef SCOREC_PCU_PCU_DEFINES_H #define SCOREC_PCU_PCU_DEFINES_H +#include +#ifndef SCOREC_NO_MPI +#include +#endif + #define PCU_SUCCESS 0 #define PCU_FAILURE -1 #ifdef __GNUC__ @@ -14,6 +19,16 @@ extern "C"{ #endif +#ifndef SCOREC_NO_MPI +typedef MPI_Comm PCU_Comm; +typedef MPI_Request PCU_Request; +#define PCU_ANY_SOURCE MPI_ANY_SOURCE +#else +typedef int PCU_Comm; +typedef int PCU_Request; +#define PCU_ANY_SOURCE -1 +#endif + struct PCU_t { void* ptr; }; @@ -22,4 +37,4 @@ struct PCU_t { } #endif -#endif // SCOREC_PCU_PCU_DEFINES_H \ No newline at end of file +#endif // SCOREC_PCU_PCU_DEFINES_H diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index 5681d0e90..652deebcd 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -38,7 +38,7 @@ static void check_rank(const pcu_mpi_t* self, int rank) PCU_ALWAYS_ASSERT(rank < pcu_mpi_size(self)); } -void pcu_mpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +void pcu_mpi_send(const pcu_mpi_t* self, pcu_message* m, PCU_Comm comm) { check_rank(self, m->peer); PCU_ALWAYS_ASSERT(comm == self->user_comm || comm == self->coll_comm); @@ -50,15 +50,24 @@ bool pcu_mpi_done(const pcu_mpi_t* self, pcu_message* m) return pcu_pmpi_done(self, m); } -bool pcu_mpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) +bool pcu_mpi_receive(const pcu_mpi_t* self, pcu_message* m, PCU_Comm comm) { - if (m->peer != MPI_ANY_SOURCE) + if (m->peer != PCU_ANY_SOURCE) check_rank(self, m->peer); return pcu_pmpi_receive(self, m, comm); } -void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi) { +void pcu_mpi_init(PCU_Comm comm, pcu_mpi_t* mpi) { pcu_pmpi_init(comm, mpi); } void pcu_mpi_finalize(pcu_mpi_t* mpi) { pcu_pmpi_finalize(mpi); } + +int pcu_mpi_split(const pcu_mpi_t* mpi, int color, int key, + PCU_Comm* newcomm) { + return pcu_pmpi_split(mpi, color, key, newcomm); +} + +int pcu_mpi_dup(const pcu_mpi_t* mpi, PCU_Comm* newcomm) { + return pcu_pmpi_dup(mpi, newcomm); +} diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 970c8a4f2..44e42c260 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -10,8 +10,8 @@ #ifndef PCU_MPI_H #define PCU_MPI_H +#include "pcu_defines.h" #include "pcu_buffer.h" -#include #ifdef __cplusplus extern "C" { @@ -20,7 +20,7 @@ extern "C" { typedef struct { pcu_buffer buffer; - MPI_Request request; + PCU_Request request; int peer; } pcu_message; @@ -29,9 +29,8 @@ void pcu_free_message(pcu_message* m); struct pcu_mpi_struct { - MPI_Comm original_comm; - MPI_Comm user_comm; - MPI_Comm coll_comm; + PCU_Comm user_comm; + PCU_Comm coll_comm; int rank; int size; }; @@ -39,11 +38,13 @@ typedef struct pcu_mpi_struct pcu_mpi_t; int pcu_mpi_size(const pcu_mpi_t*); int pcu_mpi_rank(const pcu_mpi_t*); -void pcu_mpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +void pcu_mpi_send(const pcu_mpi_t*, pcu_message* m, PCU_Comm comm); bool pcu_mpi_done(const pcu_mpi_t*, pcu_message* m); -bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); -void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); +bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, PCU_Comm comm); +void pcu_mpi_init(PCU_Comm comm, pcu_mpi_t* mpi); void pcu_mpi_finalize(pcu_mpi_t* mpi); +int pcu_mpi_split(const pcu_mpi_t* mpi, int color, int key, PCU_Comm* newcomm); +int pcu_mpi_dup(const pcu_mpi_t* mpi, PCU_Comm* newcomm); #ifdef __cplusplus } diff --git a/pcu/pcu_msg.c b/pcu/pcu_msg.c index 538310c30..58a68334b 100644 --- a/pcu/pcu_msg.c +++ b/pcu/pcu_msg.c @@ -174,7 +174,7 @@ static bool done_sending_peers(pcu_mpi_t* mpi, pcu_aa_tree t) static bool receive_global(pcu_mpi_t* mpi, pcu_msg* m) { - m->received.peer = MPI_ANY_SOURCE; + m->received.peer = PCU_ANY_SOURCE; while ( ! pcu_mpi_receive(mpi, &(m->received),mpi->user_comm)) { if (m->state == send_recv_state) diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 1db376d2c..15a8752b7 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -18,7 +18,6 @@ bool pcu_pmpi_receive2(const pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm) void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* self) { - self->original_comm = comm; MPI_Comm_dup(comm,&(self->user_comm)); MPI_Comm_dup(comm,&(self->coll_comm)); MPI_Comm_size(comm,&(self->size)); @@ -31,6 +30,16 @@ void pcu_pmpi_finalize(pcu_mpi_t* self) MPI_Comm_free(&(self->coll_comm)); } +int pcu_pmpi_split(const pcu_mpi_t *mpi, int color, int key, MPI_Comm* newcomm) +{ + return MPI_Comm_split(mpi->user_comm,color,key,newcomm); +} + +int pcu_pmpi_dup(const pcu_mpi_t *mpi, PCU_Comm* newcomm) +{ + return MPI_Comm_dup(mpi->user_comm, newcomm); +} + int pcu_pmpi_size(const pcu_mpi_t* self) { return self->size; @@ -103,3 +112,4 @@ bool pcu_pmpi_receive2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm MPI_STATUS_IGNORE); return true; } + diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index 7505aa7a4..fb9f0e134 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -16,14 +16,17 @@ #ifdef __cplusplus extern "C" { #endif -void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t *mpi); +void pcu_pmpi_init(PCU_Comm comm, pcu_mpi_t *mpi); void pcu_pmpi_finalize(pcu_mpi_t *m); int pcu_pmpi_size(const pcu_mpi_t *self); int pcu_pmpi_rank(const pcu_mpi_t *self); -void pcu_pmpi_send(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); -bool pcu_pmpi_receive(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +void pcu_pmpi_send(const pcu_mpi_t *, pcu_message *m, PCU_Comm comm); +bool pcu_pmpi_receive(const pcu_mpi_t *, pcu_message *m, PCU_Comm comm); bool pcu_pmpi_done(const pcu_mpi_t *, pcu_message *m); +int pcu_pmpi_split(const pcu_mpi_t *, int color, int key, PCU_Comm* newcomm); +int pcu_pmpi_dup(const pcu_mpi_t *, PCU_Comm* newcomm); + #ifdef __cplusplus } #endif diff --git a/pcu/pcu_pnompi.c b/pcu/pcu_pnompi.c new file mode 100644 index 000000000..900641f66 --- /dev/null +++ b/pcu/pcu_pnompi.c @@ -0,0 +1,183 @@ +/****************************************************************************** + + Copyright 2011 Scientific Computation Research Center, + Rensselaer Polytechnic Institute. All rights reserved. + + This work is open source software, licensed under the terms of the + BSD license as described in the LICENSE file in the top-level directory. + +*******************************************************************************/ +#include "pcu_pmpi.h" +#include "pcu_buffer.h" +#include +#include +#include + +void pcu_pmpi_send2(const pcu_mpi_t *, pcu_message* m, int tag, PCU_Comm comm); +bool pcu_pmpi_receive2(const pcu_mpi_t *, pcu_message* m, int tag, PCU_Comm); + +// +// ------------------------------------------------------------------ +// MPI related messages +// ------------------------------------------------------------------ + +// +// message as a doubled link list +typedef struct _NoMpiMsg +{ + char* msg; // the message + int size; // message size in byte + int tag; // tag associated with the message + int sender; // sender rank + int receiver; // receiver rank + struct _NoMpiMsg* next; // next message + struct _NoMpiMsg* prev; // previous message +} NoMpiMsg; + +static NoMpiMsg* _header = 0; // message headers + +void add_nompi_msg(void* msg, int size, int tag, int sender, int receiver) +{ + NoMpiMsg *nmsg = malloc(sizeof(NoMpiMsg)); + char* m = malloc(sizeof(char)*size); + char* mmsg = msg; + for (int i=0; imsg = m; + nmsg->size = size; + nmsg->tag = tag; + nmsg->sender = sender; + nmsg->receiver = receiver; + nmsg->next = 0; + nmsg->prev = 0; + + if (_header==0) { + _header = nmsg; + nmsg->next = nmsg; + nmsg->prev = nmsg; + } + else { + nmsg->prev = _header->prev; + _header->prev->next = nmsg; + _header->prev = nmsg; + nmsg->next = _header; + } +} + +NoMpiMsg* get_nompi_msg(int tag, int receiver) +{ + NoMpiMsg* item=_header; + if (item!=0) + { + do + { + if (item->tag == tag && item->receiver == receiver) + { + item->prev->next = item->next; + item->next->prev = item->prev; + item->next = 0; + item->prev = 0; + if (_header==item) + _header = 0; + return item; + } + item = item->next; + } while(item!=_header); + } + return 0; +} + +void free_nompi_msg(NoMpiMsg* msg) +{ + if (msg->msg) + free(msg->msg); + free(msg); +} + + +// +// ------------------------------------------------------------------- +// pcu_mpi_* functionalities +// ------------------------------------------------------------------- +// + +void pcu_pmpi_init(PCU_Comm comm, pcu_mpi_t *self) { + self->user_comm = comm+1; + self->coll_comm = comm+2; + self->size = 1; + self->rank = 0; +} + +void pcu_pmpi_finalize(pcu_mpi_t* self) { + self->user_comm = 0; + self->coll_comm = 0; +} + +int pcu_pmpi_split(const pcu_mpi_t *mpi, int c, int k, PCU_Comm *cm2) { + (void) mpi, (void) c, (void) k, (void) cm2; + return 0; +} + +int pcu_pmpi_dup(const pcu_mpi_t *mpi, PCU_Comm* newcomm) { + (void) mpi; + *newcomm = mpi->user_comm; + return 0; +} + +int pcu_pmpi_size(const pcu_mpi_t *self) { + return self->size; +} + +int pcu_pmpi_rank(const pcu_mpi_t *self) { + return self->rank; +} + +void pcu_pmpi_send(const pcu_mpi_t *self, pcu_message* m, PCU_Comm comm) { + pcu_pmpi_send2(self, m, 0, comm); +} + +void pcu_pmpi_send2(const pcu_mpi_t *self, pcu_message* m, int tag, + PCU_Comm c) { + (void) c; + if (m->buffer.size > (size_t)INT_MAX) { + fprintf(stderr, "ERROR PCU message size exceeds INT_MAX... exiting\n"); + abort(); + } + + add_nompi_msg( + m->buffer.start, (int)(m->buffer.size), tag, self->rank, m->peer + ); +} + +bool pcu_pmpi_done(const pcu_mpi_t* a, pcu_message* b) { + (void) a, (void) b; + return true; +} + +bool pcu_pmpi_receive(const pcu_mpi_t *self, pcu_message* m, PCU_Comm comm) { + return pcu_pmpi_receive2(self, m, 0, comm); +} + +bool pcu_pmpi_receive2(const pcu_mpi_t *self, pcu_message* m, int tag, + PCU_Comm comm) { + (void) comm; + NoMpiMsg* msg = get_nompi_msg(tag, self->rank); + if (msg==0) return false; + m->peer = msg->sender; + int msize = msg->size; + pcu_resize_buffer(&(m->buffer),(size_t)msize); + for (int i = 0; i < msize; i++) { + ((char*)m->buffer.start)[i] = msg->msg[i]; + } + // delete the message + free(msg->msg); + free(msg); + return true; +} + +int pcu_pmpi_barrier(PCU_Comm a) { + (void) a; + return 0; +} diff --git a/pcu/pkg_tribits.cmake b/pcu/pkg_tribits.cmake index 86278a4a2..587e0633a 100644 --- a/pcu/pkg_tribits.cmake +++ b/pcu/pkg_tribits.cmake @@ -42,12 +42,17 @@ set(SOURCES pcu_mpi.c pcu_msg.c pcu_order.c - pcu_pmpi.c pcu_util.c noto/noto_malloc.c reel/reel.c PCU.cc) +if(SCOREC_NO_MPI) + set(SOURCES ${SOURCES} pcu_pnompi.c) +else() + set(SOURCES ${SOURCES} pcu_pmpi.c) +endif() + set(HEADERS PCU_C.h pcu_io.h @@ -56,6 +61,10 @@ set(HEADERS pcu_defines.h PCU.h ) + +if(SCOREC_NO_MPI) + set(HEADERS ${HEADERS} pcu_pnompi_types.h) +endif() tribits_add_library( pcu HEADERS ${HEADERS} diff --git a/phasta/CMakeLists.txt b/phasta/CMakeLists.txt index 0a785e268..25b45144d 100644 --- a/phasta/CMakeLists.txt +++ b/phasta/CMakeLists.txt @@ -102,7 +102,6 @@ endif() # Include directories target_include_directories(ph INTERFACE $ - $ ) target_include_directories(ph PRIVATE $ diff --git a/phasta/adaptLvlSet_loop.cc b/phasta/adaptLvlSet_loop.cc index c5ac347a6..b9c2bd3c2 100644 --- a/phasta/adaptLvlSet_loop.cc +++ b/phasta/adaptLvlSet_loop.cc @@ -36,9 +36,9 @@ namespace { } int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); @@ -71,5 +71,5 @@ int main(int argc, char** argv) { Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/chef.cc b/phasta/chef.cc index 1fd034d8c..646038b14 100644 --- a/phasta/chef.cc +++ b/phasta/chef.cc @@ -29,9 +29,9 @@ namespace { /** @brief run the operations requested in "adapt.inp" */ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; pcu::Protect(); lion_set_verbosity(1); if( !PCUObj.Self() ) { @@ -69,6 +69,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/chefStream.cc b/phasta/chefStream.cc index 37f79e8c4..1229b882f 100644 --- a/phasta/chefStream.cc +++ b/phasta/chefStream.cc @@ -32,9 +32,9 @@ namespace { } int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); @@ -60,5 +60,5 @@ int main(int argc, char** argv) { Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/condense.cc b/phasta/condense.cc index 6e74a892b..c21bc00cb 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -30,7 +30,7 @@ namespace { if ( argc != 3 ) { if ( !pcu_obj->Self() ) lion_oprint(1,"Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } if ( !pcu_obj->Self() ) @@ -39,9 +39,9 @@ namespace { } int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; pcu::Protect(); lion_set_verbosity(1); checkInputs(argc,argv,&pcu_obj); @@ -65,5 +65,5 @@ int main(int argc, char** argv) { Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/cut_interface.cc b/phasta/cut_interface.cc index bf4a9e131..65bf458dc 100644 --- a/phasta/cut_interface.cc +++ b/phasta/cut_interface.cc @@ -22,7 +22,7 @@ char const* outfile; int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); lion_set_verbosity(1); if (argc < 4 || argc > 5) { lion_eprint(1,"Usage: %s \n", argv[0]); @@ -82,5 +82,5 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/migrate_interface.cc b/phasta/migrate_interface.cc index e45bc3d67..980ede4b7 100644 --- a/phasta/migrate_interface.cc +++ b/phasta/migrate_interface.cc @@ -38,7 +38,7 @@ void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) lion_eprint(1," to take model and attributes in separate files\n"); lion_eprint(1,"Usage: %s \n", argv[0]); lion_eprint(1," to take combined model and attributes file (by simTranslate)\n");} - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } if (argc == 5) { @@ -56,7 +56,7 @@ void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); @@ -90,5 +90,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/phCook.cc b/phasta/phCook.cc index c2af9bd76..6de8f399f 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -46,14 +46,12 @@ static void print_stats(const char* name, double value, pcu::PCU *pcu_obj) namespace { -pcu::PCU* createGroupComm(int splitFactor, pcu::PCU *PCUObj) +std::unique_ptr createGroupComm(int splitFactor, pcu::PCU *PCUObj) { int self = PCUObj->Self(); int groupRank = self / splitFactor; int group = self % splitFactor; - MPI_Comm groupComm; - MPI_Comm_split(PCUObj->GetMPIComm(), group, groupRank, &groupComm); - return new pcu::PCU(groupComm); + return PCUObj->Split(group, groupRank); } void loadCommon(ph::Input& in, ph::BCs& bcs, gmi_model*& g, pcu::PCU *PCUObj) @@ -270,11 +268,10 @@ namespace chef { ph::BCs bcs; loadCommon(in, bcs, g, expandedPCUObj); const int worldRank = expandedPCUObj->Self(); - pcu::PCU *groupPCUObj = createGroupComm(in.splitFactor, expandedPCUObj); + auto groupPCUObj = createGroupComm(in.splitFactor, expandedPCUObj); if ((worldRank % in.splitFactor) == 0) - originalMain(m, in, g, plan, groupPCUObj); + originalMain(m, in, g, plan, groupPCUObj.get()); if(m != nullptr) m->switchPCU(expandedPCUObj); - delete groupPCUObj; if (in.simmetrixMesh == 0) m = repeatMdsMesh(m, g, plan, in.splitFactor, expandedPCUObj); if (in.simmetrixMesh == 0 && shrinkFactor > 1){ diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 880d5d7c9..eb66229a5 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -213,7 +213,7 @@ static void postConvert(apf::Mesh2* m) int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); MS_init(); @@ -290,5 +290,5 @@ int main(int argc, char** argv) if( should_log ) Sim_logOff(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/phstream.cc b/phasta/phstream.cc index 2fe9b8d0c..9b38ce300 100644 --- a/phasta/phstream.cc +++ b/phasta/phstream.cc @@ -3,7 +3,7 @@ #include #include #include "phstream.h" -#include +#include #ifndef PHSTREAM_TIMERS_ON #define PHSTREAM_TIMERS_ON 0 @@ -11,12 +11,11 @@ namespace { inline double getTime() { - return MPI_Wtime(); + return pcu::Time(); } #if PHSTREAM_TIMERS_ON==1 inline bool isRankZero(pcu::PCU *pcu_obj) { - int rank = 0; - MPI_Comm_rank(pcu_obj->GetMPIComm(), &rank); + int rank = pcu_obj->Self(); return !rank; } #endif diff --git a/phasta/readUrPrep.cc b/phasta/readUrPrep.cc index 71eba7065..091e959e8 100644 --- a/phasta/readUrPrep.cc +++ b/phasta/readUrPrep.cc @@ -17,9 +17,9 @@ namespace { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; pcu::Protect(); gmi_register_mesh(); apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], &pcu_obj); @@ -32,6 +32,6 @@ int main(int argc, char** argv) chef::preprocess(m,ctrl); freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/phasta/threshold.cc b/phasta/threshold.cc index b619bf0a9..adc0ae740 100644 --- a/phasta/threshold.cc +++ b/phasta/threshold.cc @@ -18,9 +18,9 @@ static double process_element(apf::Vector3 x[4], double sol[4][9]) int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -59,5 +59,5 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/pumi/CMakeLists.txt b/pumi/CMakeLists.txt index 5e46b0d28..6de176274 100644 --- a/pumi/CMakeLists.txt +++ b/pumi/CMakeLists.txt @@ -40,7 +40,6 @@ add_library(pumi ${SOURCES}) # Include directories target_include_directories(pumi PUBLIC $ - $ ) target_link_libraries(pumi diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc index 0a6235b76..38a2ed5a3 100644 --- a/pumi/pumi_ghost.cc +++ b/pumi/pumi_ghost.cc @@ -10,7 +10,7 @@ #include "pumi.h" #include #include -#include +#include #include #include #include @@ -578,7 +578,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); + global_num_off_part = m->getPCU()->Add(local_num_off_part); while (global_num_off_part) { @@ -699,7 +699,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); + global_num_off_part = m->getPCU()->Add(local_num_off_part); } // while global_off_part_brg } @@ -823,7 +823,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); + global_num_off_part = m->getPCU()->Add(local_num_off_part); } if (global_num_off_part) diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index 587e87a65..017b25769 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pumi.h" -#include #include #include #include @@ -178,26 +177,6 @@ apf::Migration* getPlan(apf::Mesh* m, int num_target_part) return plan; } -void split_comm(int num_out_comm, pcu::PCU &PCUObj) -{ - int self = PCUObj.Self(); - int group_id = self % num_out_comm; - int in_group_rank = self / num_out_comm; - MPI_Comm groupComm; - MPI_Comm_split(PCUObj.GetMPIComm(), group_id, in_group_rank, &groupComm); - PCUObj.SwitchMPIComm(groupComm); -} - - -void merge_comm(MPI_Comm oldComm, pcu::PCU &PCUObj) -{ - MPI_Comm prevComm = PCUObj.GetMPIComm(); - PCUObj.SwitchMPIComm(oldComm); - MPI_Comm_free(&prevComm); -} - - - pGeom pumi_mesh_getGeom(pMesh) { return pumi::instance()->model; @@ -212,14 +191,14 @@ pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, const char* mesh_type) if (!pumi::instance()->getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->GetMPIComm(); - int num_target_part = pumi::instance()->getPCU()->Peers(); - bool isMaster = ((pumi::instance()->getPCU()->Self() % num_target_part) == 0); + bool isMaster = pumi::instance()->getPCU()->Self() == 0; pMesh m = 0; - split_comm(num_target_part, *pumi::instance()->getPCU()); + std::unique_ptr split_comm = pumi::instance()->getPCU()->Split( + pumi::instance()->getPCU()->Self(), 0 + ); if (isMaster) - m = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); - merge_comm(prevComm, *pumi::instance()->getPCU()); + m = apf::loadMdsMesh(g->getGmi(), filename, split_comm.get()); + if (m != nullptr) m->switchPCU(pumi::instance()->getPCU()); pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1, m->getPCU()); return pumi::instance()->mesh; } @@ -233,28 +212,30 @@ pMesh pumi_mesh_load(pMesh m) pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* mesh_type) { + pcu::PCU *pcu_obj = pumi::instance()->getPCU(); if (strcmp(mesh_type,"mds")) { - if (!pumi::instance()->getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Peers()>1) // do static partitioning + if (num_in_part==1 && pcu_obj->Peers()>1) // do static partitioning { - MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); - int num_target_part = pumi::instance()->getPCU()->Peers()/num_in_part; - bool isMaster = ((pumi::instance()->getPCU()->Self() % num_target_part) == 0); + int num_target_part = pcu_obj->Peers()/num_in_part; + bool isMaster = ((pcu_obj->Self() % num_target_part) == 0); pMesh m = 0; apf::Migration* plan = 0; - split_comm(num_target_part, *pumi::instance()->getPCU()); + std::unique_ptr split_comm = pcu_obj->Split( + pcu_obj->Self() % num_target_part, pcu_obj->Self() / num_target_part + ); if (isMaster) { - m = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); + m = apf::loadMdsMesh(g->getGmi(), filename, split_comm.get()); plan = getPlan(m, num_target_part); } - merge_comm(prevComm, *pumi::instance()->getPCU()); - pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part, pumi::instance()->getPCU()); + if (m != nullptr) m->switchPCU(pcu_obj); + pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part, pcu_obj); } else - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, pcu_obj); pumi_mesh_print(pumi::instance()->mesh); return pumi::instance()->mesh; } @@ -290,12 +271,12 @@ pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) else { double t0 = pcu::Time(); - MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); - int num_target_part = pumi::instance()->getPCU()->Peers(); - split_comm(num_target_part, *pumi::instance()->getPCU()); + std::unique_ptr split_comm = pumi::instance()->getPCU()->Split( + pumi::instance()->getPCU()->Self(), 0 + ); // no pmodel & remote links setup - pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); - merge_comm(prevComm, *pumi::instance()->getPCU()); + pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename, split_comm.get()); + pumi::instance()->mesh->switchPCU(pumi::instance()->getPCU()); if (!pumi::instance()->getPCU()->Self()) lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); } @@ -348,8 +329,9 @@ void pumi_mesh_setCount(pMesh m, pOwnership o) m->end(it); pumi::instance()->num_own_ent[dim] = n; } + pumi::instance()->num_global_ent[dim] = pumi::instance()->num_own_ent[dim]; } - MPI_Allreduce(pumi::instance()->num_own_ent, pumi::instance()->num_global_ent, 4, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); + m->getPCU()->Add(pumi::instance()->num_global_ent, 4); #ifdef DEBUG if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" end\n"; #endif @@ -437,12 +419,12 @@ void pumi_mesh_print (pMesh m, bool print_ent) int* global_local_entity_count = new int[4*m->getPCU()->Peers()]; int* global_own_entity_count = new int[4*m->getPCU()->Peers()]; - MPI_Allreduce(local_entity_count, global_local_entity_count, 4*m->getPCU()->Peers(), - MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - - MPI_Allreduce(own_entity_count, global_own_entity_count, 4*m->getPCU()->Peers(), - MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - + for (int i = 0; i < 4 * m->getPCU()->Peers(); ++i) { + global_local_entity_count[i] = local_entity_count[i]; + global_own_entity_count[i] = own_entity_count[i]; + } + m->getPCU()->Add(global_local_entity_count, 4*m->getPCU()->Peers()); + m->getPCU()->Add(global_own_entity_count, 4*m->getPCU()->Peers()); if (!m->getPCU()->Self()) { diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index 870e2f88e..8aaa6e6d9 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -137,7 +137,7 @@ pNumbering pumi_numbering_createProcGrp ( int* out_arr = new int[m->getPCU()->Peers()]; // out[i] has local_numOwnedPartBdryEnt of process i on all processes *in = owned_node_cnt; - MPI_Allgather(in, 1, MPI_INT, out_arr, 1, MPI_INT, m->getPCU()->GetMPIComm()); + m->getPCU()->Allgather(in, out_arr, 1); it = m->begin(dim); int nbr = 0; @@ -214,6 +214,6 @@ void pumi_numbering_print(pNumbering n, int pid) } // if shp } // for dd } // if (pid==PCU_Comm_Self()) - MPI_Barrier(MPI_COMM_WORLD); + m->getPCU()->Barrier(); } // for } diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc index 04f128bc9..cac99ddf2 100644 --- a/pumi/pumi_sys.cc +++ b/pumi/pumi_sys.cc @@ -9,7 +9,6 @@ *******************************************************************************/ #include "pumi.h" #include -#include //************************************ //************************************ @@ -32,7 +31,7 @@ int pumi_rank() void pumi_sync() { - MPI_Barrier(pumi::instance()->getPCU()->GetMPIComm()); + pumi::instance()->getPCU()->Barrier(); } #include diff --git a/python_wrappers/CMakeLists.txt b/python_wrappers/CMakeLists.txt index 1a16cd3e0..e007ec0e9 100644 --- a/python_wrappers/CMakeLists.txt +++ b/python_wrappers/CMakeLists.txt @@ -21,7 +21,6 @@ if(ENABLE_SIMMETRIX) target_include_directories(simhelper PUBLIC ${SIMMODSUITE_INCLUDE_DIR}) target_include_directories(simhelper PUBLIC $ - $ ) target_link_libraries(simhelper ${SIMMODSUITE_LIBS}) install(TARGETS simhelper diff --git a/ree/CMakeLists.txt b/ree/CMakeLists.txt index e7b935540..74169c46a 100644 --- a/ree/CMakeLists.txt +++ b/ree/CMakeLists.txt @@ -21,7 +21,6 @@ add_library(ree ${SOURCES}) # Include directories target_include_directories(ree INTERFACE $ - $ ) # Link this package to these libraries diff --git a/sam/CMakeLists.txt b/sam/CMakeLists.txt index 163747296..f990ad487 100644 --- a/sam/CMakeLists.txt +++ b/sam/CMakeLists.txt @@ -17,7 +17,6 @@ add_library(sam ${SOURCES}) # Include directories target_include_directories(sam INTERFACE $ - $ ) # Link this package to these libraries diff --git a/spr/CMakeLists.txt b/spr/CMakeLists.txt index 5a388e72f..7d9afa95b 100644 --- a/spr/CMakeLists.txt +++ b/spr/CMakeLists.txt @@ -20,7 +20,6 @@ add_library(spr ${SOURCES}) # Include directories target_include_directories(spr INTERFACE $ - $ ) # Link this package to these libraries diff --git a/stk/CMakeLists.txt b/stk/CMakeLists.txt index 4aa5970fb..d4607b971 100644 --- a/stk/CMakeLists.txt +++ b/stk/CMakeLists.txt @@ -61,7 +61,6 @@ target_link_libraries(apf_stk PUBLIC ${Shards_LIBRARIES}) # Include directories target_include_directories(apf_stk INTERFACE $ - $ ) target_include_directories(apf_stk PUBLIC ${Shards_INCLUDE_DIRS} diff --git a/test/1d.cc b/test/1d.cc index 654eab63d..272ad0977 100644 --- a/test/1d.cc +++ b/test/1d.cc @@ -64,9 +64,9 @@ int main(int argc, char** argv) * 2 - model output name * 3 = mesh output name **/ PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_model* g; apf::Mesh2* m; @@ -80,5 +80,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc index ece447aee..86c8f0a8a 100644 --- a/test/H1Shapes.cc +++ b/test/H1Shapes.cc @@ -27,9 +27,9 @@ void testH1( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); @@ -73,7 +73,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } void E_exact(const apf::Vector3& x, apf::Vector3& value, int p) diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc index 9f7525769..d26567c52 100644 --- a/test/L2Shapes.cc +++ b/test/L2Shapes.cc @@ -30,9 +30,9 @@ void testL2writeNative( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); @@ -62,7 +62,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } void E_exact(const apf::Vector3& x, apf::Vector3& value, int p) diff --git a/test/align.cc b/test/align.cc index 83166344d..3f0eb3760 100644 --- a/test/align.cc +++ b/test/align.cc @@ -89,14 +89,14 @@ void testTetTri(pcu::PCU *PCUObj) } int main() { - MPI_Init(0,0); + pcu::Init(0,0); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_register_null(); testTriEdge(&pcu_obj); testTetEdge(&pcu_obj); testTetTri(&pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/aniso_ma_test.cc b/test/aniso_ma_test.cc index 499dbfbf5..f1d99a360 100644 --- a/test/aniso_ma_test.cc +++ b/test/aniso_ma_test.cc @@ -44,9 +44,9 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; bool logInterpolation = atoi(argv[3]) > 0 ? true : false; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); @@ -68,6 +68,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ansys.cc b/test/ansys.cc index e9573b084..3a0ecd833 100644 --- a/test/ansys.cc +++ b/test/ansys.cc @@ -7,14 +7,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); @@ -25,7 +25,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/assert_timing.cc b/test/assert_timing.cc index 699f0441d..18f7bb5fa 100644 --- a/test/assert_timing.cc +++ b/test/assert_timing.cc @@ -29,9 +29,9 @@ double check_pcu_assert() { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 2); int opt = atoi(argv[1]); - MPI_Init(0,0); + pcu::Init(0,0); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); /* i'm avoiding conditionals inside for loops b/c i'm paranoid about the timings even though timings @@ -43,6 +43,6 @@ int main(int argc, char** argv) { for (int i = 0; i < 5; ++i) printf("pcu assert in %f seconds\n", check_pcu_assert()); } - MPI_Finalize(); + pcu::Finalize(); return 0; } diff --git a/test/balance.cc b/test/balance.cc index 68efd60e4..ed791522a 100644 --- a/test/balance.cc +++ b/test/balance.cc @@ -9,9 +9,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_register_mesh(); //load model and mesh @@ -28,5 +28,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/bezierElevation.cc b/test/bezierElevation.cc index 2bb018624..f8b84dfb2 100644 --- a/test/bezierElevation.cc +++ b/test/bezierElevation.cc @@ -397,13 +397,13 @@ void testTetElevation(pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); testEdgeElevation(&pcu_obj); testTriElevation(&pcu_obj); testTetElevation(&pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/bezierMesh.cc b/test/bezierMesh.cc index fbf317c3f..e59ba7564 100644 --- a/test/bezierMesh.cc +++ b/test/bezierMesh.cc @@ -583,13 +583,13 @@ void test3DFull(pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); test2D(&pcu_obj); test3DBlended(&pcu_obj); test3DFull(&pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/bezierMisc.cc b/test/bezierMisc.cc index 2f3b389c2..5ca394257 100644 --- a/test/bezierMisc.cc +++ b/test/bezierMisc.cc @@ -165,13 +165,13 @@ void testMatrixInverse(){ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); testNodeIndexing(); testMatrixInverse(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/bezierRefine.cc b/test/bezierRefine.cc index f413108b2..88aae0fe4 100644 --- a/test/bezierRefine.cc +++ b/test/bezierRefine.cc @@ -309,12 +309,12 @@ void test3D(pcu::PCU *PCUObj) } int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); test2D(&pcu_obj); test3D(&pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/bezierShapeEval.cc b/test/bezierShapeEval.cc index ba6b5136d..5a9f9d754 100644 --- a/test/bezierShapeEval.cc +++ b/test/bezierShapeEval.cc @@ -25,9 +25,9 @@ static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PC int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 7 ) { if ( !PCUObj.Self() ) { @@ -37,7 +37,7 @@ int main(int argc, char** argv) printf(" can be -1, 0, 1, 2 (-1 means no blending)\n"); printf(" inquiry point in the parent entity)\n"); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -81,7 +81,7 @@ int main(int argc, char** argv) } } - MPI_Finalize(); + pcu::Finalize(); } static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj) diff --git a/test/bezierSubdivision.cc b/test/bezierSubdivision.cc index 308722fe5..9a16bd37d 100644 --- a/test/bezierSubdivision.cc +++ b/test/bezierSubdivision.cc @@ -480,14 +480,14 @@ void testTetSubdivision1(pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); testEdgeSubdivision(&pcu_obj); testTriSubdivision1(&pcu_obj); testTriSubdivision4(&pcu_obj); testTetSubdivision1(&pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/bezierValidity.cc b/test/bezierValidity.cc index 9f627c3e0..9a580b157 100644 --- a/test/bezierValidity.cc +++ b/test/bezierValidity.cc @@ -405,12 +405,12 @@ void test3D(pcu::PCU *PCUObj) } int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); test2D(&pcu_obj); test3D(&pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/box.cc b/test/box.cc index a5ed9b0dc..a4694ebc0 100644 --- a/test/box.cc +++ b/test/box.cc @@ -36,7 +36,7 @@ void verifyArgs(int argc, char** argv, pcu::PCU *PCUObj) printf(" is simplical mesh\n"); printf(" .dmg model file name\n"); printf(" .smb mesh file name\n"); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } } @@ -59,9 +59,9 @@ void getArgs(char** argv) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); verifyArgs(argc, argv, &pcu_obj); if(pcu_obj.Peers()>1) { @@ -79,5 +79,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/cap2vtk.cc b/test/cap2vtk.cc index a2891e4ec..1b2c696e7 100644 --- a/test/cap2vtk.cc +++ b/test/cap2vtk.cc @@ -40,9 +40,9 @@ using namespace CreateMG::Geometry; int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; if (argc != 3) { if(0==PCUObj.Self()) @@ -139,5 +139,5 @@ int main(int argc, char** argv) gmi_cap_stop(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/capCheckParam.cc b/test/capCheckParam.cc index df686c16e..a349bff2c 100644 --- a/test/capCheckParam.cc +++ b/test/capCheckParam.cc @@ -27,11 +27,12 @@ void checkParametrization(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* int main(int argc, char** argv) { - MPI_Init(&argc, &argv); - pcu::PCU *PCUobj = new pcu::PCU(MPI_COMM_WORLD); + pcu::Init(&argc, &argv); + { // pcu object scope + pcu::PCU PCUobj; if (argc != 2) { - if (PCUobj->Self() == 0) + if (PCUobj.Self() == 0) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; } @@ -111,8 +112,8 @@ int main(int argc, char** argv) // check parametrization using capstone apis checkParametrization(m, g); - delete PCUobj; - MPI_Finalize(); + } // pcu object scope + pcu::Finalize(); } void checkParametrization(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb) diff --git a/test/capGeomTest.cc b/test/capGeomTest.cc index bfbd6efbd..25f681fa2 100644 --- a/test/capGeomTest.cc +++ b/test/capGeomTest.cc @@ -48,8 +48,9 @@ void visualizeEdges(gmi_model* model, int n, const char* fileName, pcu::PCU *PCU int main(int argc, char** argv) { - MPI_Init(&argc, &argv); - pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); + pcu::Init(&argc, &argv); + { // pcu object scope + pcu::PCU PCUObj; gmi_register_mesh(); gmi_register_null(); @@ -117,7 +118,7 @@ int main(int argc, char** argv) gmi_register_cap(); - apf::Mesh2* mesh0 = apf::createMesh(m,g,PCUObj); + apf::Mesh2* mesh0 = apf::createMesh(m,g,&PCUObj); apf::writeVtkFiles("mesh_no_param", mesh0); gmi_model* model = gmi_import_cap(g); @@ -138,7 +139,7 @@ int main(int argc, char** argv) while( (ge = gmi_next(model, gi)) ){ std::stringstream name_str; name_str << "face_" << gmi_tag(model, ge) << "_mesh"; - visualizeFace(model, ge, 100, 100, name_str.str().c_str(), PCUObj); + visualizeFace(model, ge, 100, 100, name_str.str().c_str(), &PCUObj); } gmi_end(model, gi); @@ -146,7 +147,7 @@ int main(int argc, char** argv) printf("creating mesh with param field\n"); - apf::Mesh2* mesh = apf::createMesh(m,g,PCUObj); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); apf::Field* pf = apf::createFieldOn(mesh, "param_field", apf::VECTOR); apf::Field* idf = apf::createFieldOn(mesh, "id", apf::SCALAR); apf::MeshEntity* e; @@ -164,8 +165,8 @@ int main(int argc, char** argv) gmi_cap_stop(); - delete PCUObj; - MPI_Finalize(); + } // pcu object scope + pcu::Finalize(); } void printInfo(gmi_model* model, int dim) diff --git a/test/capVol.cc b/test/capVol.cc index 3f07926d0..7e44c2e36 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -30,7 +30,7 @@ namespace { void myExit(int exit_code = EXIT_SUCCESS) { gmi_cap_stop(); - MPI_Finalize(); + pcu::Finalize(); exit(exit_code); } @@ -75,9 +75,9 @@ void printUsage(char *argv0) { int main(int argc, char** argv) { // Initialize parallelism. - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; // Initialize logging. lion_set_stdout(stdout); @@ -272,5 +272,5 @@ int main(int argc, char** argv) { // Exit calls. gmi_cap_stop(); } - MPI_Finalize(); + pcu::Finalize(); } \ No newline at end of file diff --git a/test/cap_closestPoint.cc b/test/cap_closestPoint.cc index 778464e88..4db2b59c2 100644 --- a/test/cap_closestPoint.cc +++ b/test/cap_closestPoint.cc @@ -7,8 +7,9 @@ #include int main (int argc, char* argv[]) { - MPI_Init(&argc, &argv); - pcu::PCU* PCUObj = new pcu::PCU(MPI_COMM_WORLD); + pcu::Init(&argc, &argv); + { // pcu object scope + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_cap(); @@ -19,7 +20,7 @@ int main (int argc, char* argv[]) { "Mesh Database : Create", "Attribution Database : Create"); cs.load_files(v_string(1, creFile)); // 2. CreateMesh. - apf::Mesh2* m = apf::createMesh(cs.get_mesh(), cs.get_geometry(), PCUObj); + apf::Mesh2* m = apf::createMesh(cs.get_mesh(), cs.get_geometry(), &PCUObj); PCU_ALWAYS_ASSERT(m->canGetClosestPoint()); @@ -34,6 +35,6 @@ int main (int argc, char* argv[]) { PCU_ALWAYS_ASSERT((to - apf::Vector3(0.5, 0.34, 0.5)).getLength() < 0.0001); apf::destroyMesh(m); - delete PCUObj; - MPI_Finalize(); + } // pcu object scope + pcu::Finalize(); } diff --git a/test/cap_inClosureOf.cc b/test/cap_inClosureOf.cc index 8cf88ee88..63ebb828b 100644 --- a/test/cap_inClosureOf.cc +++ b/test/cap_inClosureOf.cc @@ -7,8 +7,9 @@ #include int main (int argc, char* argv[]) { - MPI_Init(&argc, &argv); - pcu::PCU *PCUObj = new pcu::PCU(MPI_COMM_WORLD); + pcu::Init(&argc, &argv); + { // pcu object scope + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_cap(); @@ -19,7 +20,7 @@ int main (int argc, char* argv[]) { "Mesh Database : Create", "Attribution Database : Create"); cs.load_files(v_string(1, creFile)); // 2. CreateMesh. - apf::Mesh2* m = apf::createMesh(cs.get_mesh(), cs.get_geometry(), PCUObj); + apf::Mesh2* m = apf::createMesh(cs.get_mesh(), cs.get_geometry(), &PCUObj); // 3. Get region 1 ModelEntity*. apf::ModelEntity* rgn = m->findModelEntity(3, 1); PCU_ALWAYS_ASSERT(rgn); @@ -44,6 +45,6 @@ int main (int argc, char* argv[]) { PCU_ALWAYS_ASSERT(m->isInClosureOf(m->findModelEntity(0, 8), f1)); apf::destroyMesh(m); - delete PCUObj; - MPI_Finalize(); + } // pcu object scope + pcu::Finalize(); } diff --git a/test/cgns.cc b/test/cgns.cc index d84dc96c5..66ea7d439 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -478,9 +478,9 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: int main(int argc, char **argv) { #ifdef HAVE_CGNS - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; pumi_load_pcu(&PCUObj); lion_set_verbosity(1); bool additionalTests = false; @@ -488,7 +488,7 @@ int main(int argc, char **argv) { if (!PCUObj.Self()) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); return -1; } @@ -500,7 +500,7 @@ int main(int argc, char **argv) { if (!PCUObj.Self()) printf("Usage: %s additional\n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); return -1; } @@ -509,7 +509,7 @@ int main(int argc, char **argv) { if (!PCUObj.Self()) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); return -1; } @@ -546,7 +546,7 @@ int main(int argc, char **argv) } // } - MPI_Finalize(); + pcu::Finalize(); return 0; #else PCU_ALWAYS_ASSERT_VERBOSE(true == false, diff --git a/test/classifyThenAdapt.cc b/test/classifyThenAdapt.cc index 081ddf5c7..b957ac373 100644 --- a/test/classifyThenAdapt.cc +++ b/test/classifyThenAdapt.cc @@ -125,9 +125,9 @@ void printClassCounts(apf::Mesh* m) { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_register_null(); @@ -149,5 +149,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/collapse.cc b/test/collapse.cc index 2ddced964..818acb4f0 100644 --- a/test/collapse.cc +++ b/test/collapse.cc @@ -32,7 +32,7 @@ namespace { printf("Usage: mpirun -np %s \n" "Reduce the part count of mesh from inPartCount to inPartCount/factor.\n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -44,9 +44,9 @@ namespace { } int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); pcu::Protect(); #ifdef HAVE_SIMMETRIX @@ -67,5 +67,5 @@ int main(int argc, char** argv) { Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/construct.cc b/test/construct.cc index cfb5597bd..974e012ca 100644 --- a/test/construct.cc +++ b/test/construct.cc @@ -10,9 +10,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -46,6 +46,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index a4d17cf1a..47242ff57 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -12,9 +12,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; pumi_load_pcu(&pcu_obj); lion_set_verbosity(1); gmi_register_mesh(); @@ -93,5 +93,5 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(pm); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/construct_bottom_up.cc b/test/construct_bottom_up.cc index f982dff88..283b0b049 100644 --- a/test/construct_bottom_up.cc +++ b/test/construct_bottom_up.cc @@ -137,9 +137,9 @@ const int tet_info[11][6] = { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.Peers() == 1, "Not implemented in parallel!"); @@ -249,6 +249,6 @@ int main(int argc, char** argv) outMesh->destroyNative(); apf::destroyMesh(outMesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/convert.cc b/test/convert.cc index fbca0d5a2..4ee4b4d0a 100644 --- a/test/convert.cc +++ b/test/convert.cc @@ -433,9 +433,9 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); //for fancy BL/extrusion queries @@ -526,5 +526,5 @@ int main(int argc, char** argv) if( should_log ) Sim_logOff(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/crack_test.cc b/test/crack_test.cc index cbb7f1ef5..f5b2c857f 100644 --- a/test/crack_test.cc +++ b/test/crack_test.cc @@ -44,15 +44,15 @@ void bCurver(const char* modelFile, const char* meshFile, int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; if (argc < 2) { if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } const char* modelFile = argv[1]; @@ -359,5 +359,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/create_mis.cc b/test/create_mis.cc index bedc28f0d..444cfc0bb 100644 --- a/test/create_mis.cc +++ b/test/create_mis.cc @@ -9,14 +9,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -61,6 +61,6 @@ int main(int argc, char** argv) apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/curve_to_bezier.cc b/test/curve_to_bezier.cc index fb9fadc0c..70e686fc1 100644 --- a/test/curve_to_bezier.cc +++ b/test/curve_to_bezier.cc @@ -14,9 +14,9 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); MS_init(); SimModel_start(); @@ -25,14 +25,14 @@ int main(int argc, char** argv) if ( argc != 5 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } int order = atoi(argv[3]); if(order < 1 || order > 6){ if ( !pcu_obj.Self() ) printf("Only 1st to 6th order supported\n"); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -55,5 +55,5 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/curvetest.cc b/test/curvetest.cc index 2b4b51750..f1d5526de 100644 --- a/test/curvetest.cc +++ b/test/curvetest.cc @@ -190,9 +190,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); MS_init(); SimModel_start(); @@ -214,5 +214,5 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/degenerateSurfs.cc b/test/degenerateSurfs.cc index 71ece5203..c36a59995 100644 --- a/test/degenerateSurfs.cc +++ b/test/degenerateSurfs.cc @@ -23,7 +23,7 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) if ( argc != 5 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -35,9 +35,9 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==5); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -72,6 +72,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/describe.cc b/test/describe.cc index 4ed8b59eb..e28e6da8f 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -44,9 +44,9 @@ static void list_tags(apf::Mesh* m) int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -71,5 +71,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/dg_ma_test.cc b/test/dg_ma_test.cc index 5e2b7b50a..a33bd5b4d 100644 --- a/test/dg_ma_test.cc +++ b/test/dg_ma_test.cc @@ -39,9 +39,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -77,6 +77,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/elmBalance.cc b/test/elmBalance.cc index be0d2a2bf..157ecf396 100644 --- a/test/elmBalance.cc +++ b/test/elmBalance.cc @@ -27,14 +27,14 @@ apf::MeshTag* setWeights(apf::Mesh* m) { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -68,5 +68,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/embedded_edges.cc b/test/embedded_edges.cc index fcfa4b565..f84caa438 100644 --- a/test/embedded_edges.cc +++ b/test/embedded_edges.cc @@ -15,9 +15,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==2); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -53,6 +53,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/extrude.cc b/test/extrude.cc index 85e3aeb59..f011e2709 100644 --- a/test/extrude.cc +++ b/test/extrude.cc @@ -9,14 +9,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -40,5 +40,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fieldReduce.cc b/test/fieldReduce.cc index 4bc94ac65..29ee782f5 100644 --- a/test/fieldReduce.cc +++ b/test/fieldReduce.cc @@ -151,7 +151,7 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) if ( argc != 3 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -164,13 +164,13 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); bool failflag = false; { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); - MPI_Comm_rank(PCUObj.GetMPIComm(), &myrank); - MPI_Comm_size(PCUObj.GetMPIComm(), &commsize); + myrank = PCUObj.Self(); + commsize = PCUObj.Peers(); #ifdef HAVE_SIMMETRIX MS_init(); SimModel_start(); @@ -197,7 +197,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); return failflag; diff --git a/test/field_io.cc b/test/field_io.cc index 44c2c8305..fd581efbb 100644 --- a/test/field_io.cc +++ b/test/field_io.cc @@ -9,9 +9,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); { @@ -47,5 +47,5 @@ int main(int argc, char** argv) apf::destroyMesh(m); } } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fixDisconnected.cc b/test/fixDisconnected.cc index 5901e84b5..7ef93e957 100644 --- a/test/fixDisconnected.cc +++ b/test/fixDisconnected.cc @@ -10,14 +10,14 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -29,5 +29,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fixlayer.cc b/test/fixlayer.cc index f018e51c0..c0db6f462 100644 --- a/test/fixlayer.cc +++ b/test/fixlayer.cc @@ -14,9 +14,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -40,7 +40,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fixshape.cc b/test/fixshape.cc index fc32ab791..830d316e4 100644 --- a/test/fixshape.cc +++ b/test/fixshape.cc @@ -14,9 +14,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -40,7 +40,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fusion.cc b/test/fusion.cc index f9fc9e216..e4a6ee7b1 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -154,9 +154,9 @@ struct GroupCode : public Parma_GroupCode int main( int argc, char* argv[]) { PCU_ALWAYS_ASSERT(argc==2); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); GroupCode code; code.model = makeModel(); @@ -171,5 +171,5 @@ int main( int argc, char* argv[]) code.mesh->destroyNative(); apf::destroyMesh(code.mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fusion2.cc b/test/fusion2.cc index d252af990..faeb0c2cc 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -137,9 +137,9 @@ static void globalCode(apf::Mesh2* m) int main( int argc, char* argv[]) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); PCU_ALWAYS_ASSERT(PCUObj.Peers() == 2); gmi_register_null(); @@ -150,5 +150,5 @@ int main( int argc, char* argv[]) Parma_SplitPartition(code.mesh, groupSize, code, &PCUObj); globalCode(code.mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/fusion3.cc b/test/fusion3.cc index ee3e562c1..d4567725e 100644 --- a/test/fusion3.cc +++ b/test/fusion3.cc @@ -281,9 +281,9 @@ class Vortex : public ma::AnisotropicFunction int main(int argc, char * argv[]) { PCU_ALWAYS_ASSERT(argc==2); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_model* model = makeModel(); gmi_write_dmg(model, "made.dmg"); @@ -298,5 +298,5 @@ int main(int argc, char * argv[]) // to do apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/gap.cc b/test/gap.cc index 36554ffd7..699cb92b0 100644 --- a/test/gap.cc +++ b/test/gap.cc @@ -28,14 +28,14 @@ namespace { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 5); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -69,5 +69,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/generate.cc b/test/generate.cc index 7f2c4f053..5e76958e5 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -66,7 +66,7 @@ void messageHandler(int type, const char* msg) case Sim_ErrorMsg: if(!globalPCU->Self()) fprintf(stdout, "Error SimModeler %s ... exiting\n", msg); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_SUCCESS); break; default: @@ -317,9 +317,9 @@ void simStop() { int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; globalPCU = &PCUObj; lion_set_verbosity(1); pcu::Protect(); @@ -359,5 +359,5 @@ int main(int argc, char** argv) simStop(); Sim_logOff(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ghost.cc b/test/ghost.cc index 190feb5eb..d2ebb9808 100644 --- a/test/ghost.cc +++ b/test/ghost.cc @@ -22,7 +22,7 @@ namespace { if ( argc != 4 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -76,9 +76,9 @@ namespace { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv,&PCUObj); @@ -89,5 +89,5 @@ int main(int argc, char** argv) Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ghostEdge.cc b/test/ghostEdge.cc index 6f4d3ae40..d6883304b 100644 --- a/test/ghostEdge.cc +++ b/test/ghostEdge.cc @@ -49,9 +49,9 @@ namespace { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); @@ -60,5 +60,5 @@ int main(int argc, char** argv) m->writeNative(argv[3]); freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ghostMPAS.cc b/test/ghostMPAS.cc index 8af944c6a..568090e29 100644 --- a/test/ghostMPAS.cc +++ b/test/ghostMPAS.cc @@ -51,9 +51,9 @@ namespace { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); @@ -62,5 +62,5 @@ int main(int argc, char** argv) m->writeNative(argv[3]); freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/gmsh.cc b/test/gmsh.cc index 2161ec764..f259b3cac 100644 --- a/test/gmsh.cc +++ b/test/gmsh.cc @@ -9,9 +9,9 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !PCUObj.Self() ) @@ -23,7 +23,7 @@ int main(int argc, char** argv) "the mesh.\n" "When a **gmsh v4** .msh is passed in, a topological model will be created " "from the geometric model entities defined in the gmsh input file.\n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); @@ -55,6 +55,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/graphdist.cc b/test/graphdist.cc index 3d9e96041..61fe1bc37 100644 --- a/test/graphdist.cc +++ b/test/graphdist.cc @@ -9,14 +9,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -32,5 +32,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/hierarchic.cc b/test/hierarchic.cc index 26bccc7de..7c450eeda 100644 --- a/test/hierarchic.cc +++ b/test/hierarchic.cc @@ -192,9 +192,9 @@ void test(apf::Mesh* m, int p_order) { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); PCU_ALWAYS_ASSERT(! pcu_obj.Self()); gmi_register_mesh(); @@ -206,5 +206,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc index db6a71ffb..e299c0c56 100644 --- a/test/highOrderSizeFields.cc +++ b/test/highOrderSizeFields.cc @@ -35,9 +35,9 @@ void testAdapt( int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX @@ -69,7 +69,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - MPI_Finalize(); + pcu::Finalize(); } void computeSizesFrames( diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index 57eb318d5..638f426e3 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -41,9 +41,9 @@ int main(int argc, char** argv) { const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -91,7 +91,7 @@ int main(int argc, char** argv) MS_exit(); SimModel_stop(); #endif - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/icesheet.cc b/test/icesheet.cc index a05b4061d..8115dec73 100644 --- a/test/icesheet.cc +++ b/test/icesheet.cc @@ -481,9 +481,9 @@ int main(int argc, char** argv) return 0; } - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -525,5 +525,5 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/inClosureOf_test.cc b/test/inClosureOf_test.cc index f76448979..958a22541 100644 --- a/test/inClosureOf_test.cc +++ b/test/inClosureOf_test.cc @@ -18,9 +18,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==2); const char* modelFile = argv[1]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -94,6 +94,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/intrude.cc b/test/intrude.cc index bf1fbe332..48ecf7546 100644 --- a/test/intrude.cc +++ b/test/intrude.cc @@ -9,14 +9,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -42,5 +42,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/loadPart.cc b/test/loadPart.cc index 32d4067f7..9c8f12744 100644 --- a/test/loadPart.cc +++ b/test/loadPart.cc @@ -10,9 +10,9 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); PCU_ALWAYS_ASSERT(PCUObj.Peers() == 1); if ( argc != 4 ) { @@ -20,7 +20,7 @@ int main(int argc, char** argv) printf("Load a single part from a partitioned mesh and " "write it as a serial part.\n" "Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); @@ -33,7 +33,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ma_insphere.cc b/test/ma_insphere.cc index 4b990a705..bab162b28 100644 --- a/test/ma_insphere.cc +++ b/test/ma_insphere.cc @@ -7,7 +7,7 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); // Test determinant functions double input[4][4] = { @@ -30,7 +30,7 @@ int main(int argc, char** argv) // Test insphere (create a mesh with one tet) { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); apf::Vector3 a(0, 0, 0); apf::Vector3 b(-6, 0, 0); @@ -58,5 +58,5 @@ int main(int argc, char** argv) apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ma_test.cc b/test/ma_test.cc index 86100546b..8a98f271d 100644 --- a/test/ma_test.cc +++ b/test/ma_test.cc @@ -43,9 +43,9 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; const char* layerTagString = (argc==4) ? argv[3] : ""; const double adaptRefineFactor = (argc==5) ? atoi(argv[4]) : 3; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -81,6 +81,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ma_test_analytic_model.cc b/test/ma_test_analytic_model.cc index 1a03f233d..bd7cf7979 100644 --- a/test/ma_test_analytic_model.cc +++ b/test/ma_test_analytic_model.cc @@ -169,9 +169,9 @@ apf::Mesh2* createSphereMesh(pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); apf::Mesh2* m = createSphereMesh(&PCUObj); @@ -187,5 +187,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc index df5692c48..ed82c9a58 100644 --- a/test/makeAllCavities.cc +++ b/test/makeAllCavities.cc @@ -61,13 +61,13 @@ static apf::MeshTag* tagMesh( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; if (PCUObj.Peers() > 1) { printf("%s should only be used for serial (single part) meshes!\n", argv[0]); printf("use the serialize utility to get a serial mesh, and retry!\n"); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } if (argc != 6) { @@ -87,7 +87,7 @@ int main(int argc, char** argv) printf("fb: creates all face cavities classified on boundary\n"); printf("ls: get a list from user and creates cavities for that list\n"); printf("tagname: creates cavities for all entities that have tagname\n"); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -166,7 +166,7 @@ int main(int argc, char** argv) tag = m->findTag(mode.c_str()); if (!tag) { printf("tag with name %s was not found on the mesh. Aborting!\n", mode.c_str()); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } } @@ -254,7 +254,7 @@ int main(int argc, char** argv) #endif } - MPI_Finalize(); + pcu::Finalize(); } static void safe_mkdir( diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index a135f0703..314908c35 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -611,7 +611,7 @@ fh = header file (there is only one for all processes), containing: **/ std::vector readHeader(std::ifstream& fh, pcu::PCU *PCUObj) { rewindStream(fh); - const int self = PCUObj->Self();; + const int self = PCUObj->Self(); bool ret = seekPart(fh, std::to_string(self)); PCU_ALWAYS_ASSERT(ret); auto blockInfo = readTopoBlockInfo(fh); @@ -743,9 +743,9 @@ void readMesh(const char* meshfilename, int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); int noVerify=0; // maintain default of verifying if not explicitly requesting it off if( argc < 11 ) { @@ -829,5 +829,5 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/measureAnisoStats.cc b/test/measureAnisoStats.cc index c748303b2..198a8c916 100644 --- a/test/measureAnisoStats.cc +++ b/test/measureAnisoStats.cc @@ -41,9 +41,9 @@ void getStats( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (argc < 5) { if (PCUObj.Self() == 0) { @@ -52,7 +52,7 @@ int main(int argc, char** argv) printf("USAGE2: %s " "\n", argv[0]); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -89,7 +89,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } void safe_mkdir(const char* path) diff --git a/test/measureIsoStats.cc b/test/measureIsoStats.cc index 4ebacec5e..cef3d410f 100644 --- a/test/measureIsoStats.cc +++ b/test/measureIsoStats.cc @@ -42,9 +42,9 @@ void getStats( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (argc < 4) { if (PCUObj.Self() == 0) { @@ -53,7 +53,7 @@ int main(int argc, char** argv) printf("USAGE2: %s " "\n", argv[0]); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -89,7 +89,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); #endif } - MPI_Finalize(); + pcu::Finalize(); } void safe_mkdir(const char* path) diff --git a/test/mixedNumbering.cc b/test/mixedNumbering.cc index 9017b454d..22ff425f6 100644 --- a/test/mixedNumbering.cc +++ b/test/mixedNumbering.cc @@ -47,9 +47,9 @@ static void write_output(apf::Mesh* m, const char* out) { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_register_mesh(); apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); @@ -59,5 +59,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/mkmodel.cc b/test/mkmodel.cc index 3d5e063f3..04f212876 100644 --- a/test/mkmodel.cc +++ b/test/mkmodel.cc @@ -9,15 +9,15 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 3 ) { if ( !pcu_obj.Self() ) printf("Create a discrete geometric model from a mesh\n" "Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); @@ -27,6 +27,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/mktopomodel.cc b/test/mktopomodel.cc index e0b64d360..3ac9d39c8 100644 --- a/test/mktopomodel.cc +++ b/test/mktopomodel.cc @@ -9,15 +9,15 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 3 ) { if ( !pcu_obj.Self() ) printf("Create a topological geometric model from a mesh\n" "Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); @@ -29,6 +29,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/modelInfo.cc b/test/modelInfo.cc index f35450f7d..bced288e7 100644 --- a/test/modelInfo.cc +++ b/test/modelInfo.cc @@ -10,18 +10,17 @@ #endif #include //exit and exit_failure #include -#include int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 2 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -78,6 +77,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/moving.cc b/test/moving.cc index 7e297fe32..9ebe5aa33 100644 --- a/test/moving.cc +++ b/test/moving.cc @@ -18,9 +18,9 @@ static void writeStep(apf::Mesh* m, int i) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 3 ) { fprintf(stderr, "Usage: %s \n", argv[0]); @@ -67,6 +67,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc index 534f4d04b..c980759e2 100644 --- a/test/nedelecShapes.cc +++ b/test/nedelecShapes.cc @@ -26,9 +26,9 @@ void testNedelec( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(0); @@ -67,7 +67,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } void E_exact(const apf::Vector3& x, apf::Vector3& value, int p) diff --git a/test/nektar_align.cc b/test/nektar_align.cc index 62010da87..9fb33e587 100644 --- a/test/nektar_align.cc +++ b/test/nektar_align.cc @@ -108,9 +108,9 @@ static void alignForNektar(apf::Mesh2* m) int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -132,5 +132,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/neper.cc b/test/neper.cc index 8be481036..224ee4a83 100644 --- a/test/neper.cc +++ b/test/neper.cc @@ -7,14 +7,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -24,6 +24,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/newdim.cc b/test/newdim.cc index 29b68d160..654be4268 100644 --- a/test/newdim.cc +++ b/test/newdim.cc @@ -6,9 +6,9 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_null(); gmi_model* model = gmi_load(".null"); @@ -40,7 +40,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/osh2smb.cc b/test/osh2smb.cc index c45eb6777..6855b4a7f 100644 --- a/test/osh2smb.cc +++ b/test/osh2smb.cc @@ -14,9 +14,9 @@ #include int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (argc != 4) { if (PCUObj.Self() == 0) { @@ -24,7 +24,7 @@ int main(int argc, char** argv) { std::cout << "usage: osh2smb in.osh in.dmg out.smb\n"; std::cout << " or: osh2smb (usage)\n"; } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -41,5 +41,5 @@ int main(int argc, char** argv) { apf::destroyMesh(am); } } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index 72a9a7d20..640a202bf 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -46,9 +46,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -102,6 +102,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/poisson.cc b/test/poisson.cc index 1d97d048c..859a601f2 100644 --- a/test/poisson.cc +++ b/test/poisson.cc @@ -250,14 +250,14 @@ void test(int dim, int p, pcu::PCU *PCUObj) { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); PCU_ALWAYS_ASSERT(! pcu_obj.Self()); int dim = atoi(argv[1]); int p = atoi(argv[2]); test(dim, p, &pcu_obj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/print_pumipic_partition.cc b/test/print_pumipic_partition.cc index 371a43614..f68d0d6f4 100644 --- a/test/print_pumipic_partition.cc +++ b/test/print_pumipic_partition.cc @@ -18,20 +18,20 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 5 && argc != 6) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } if (pcu_obj.Peers() > 1) { if ( !pcu_obj.Self() ) printf("This tool must be run in serial.\n"); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -81,5 +81,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ptnParma.cc b/test/ptnParma.cc index d43a22a9f..689e25b2c 100644 --- a/test/ptnParma.cc +++ b/test/ptnParma.cc @@ -142,7 +142,7 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) if ( !PCUObj->Self() ) printf("Usage: %s " " <0:global|1:local>\n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -162,9 +162,9 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if( !PCUObj.Self() ) lion_oprint(1, "PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); @@ -189,5 +189,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/pumi.cc b/test/pumi.cc index 59aea11fd..220d8d0d7 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -11,7 +11,6 @@ #include #include #include -#include const char* modelFile = 0; const char* meshFile = 0; @@ -42,7 +41,7 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) if ( argc < 4 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -86,9 +85,9 @@ void TEST_FIELD(pMesh m); int main(int argc, char** argv) //********************************************************* { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; pumi_load_pcu(&PCUObj); lion_set_verbosity(1); pumi_printSys(); @@ -252,7 +251,7 @@ int main(int argc, char** argv) pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem); } - MPI_Finalize(); + pcu::Finalize(); } void TEST_MESH(pMesh m) @@ -791,7 +790,7 @@ void TEST_GHOSTING(pMesh m) pumi_ghost_create(m, ghosting_plan); int total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; - MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); + total_mcount_diff = m->getPCU()->Add(mcount_diff); if (!pumi_rank()) std::cout<<"\n[test_pumi] element-wise pumi_ghost_create: #ghost increase="<getPCU()->GetMPIComm()); + total_mcount_diff = m->getPCU()->Add(mcount_diff); if (!pumi_rank()) std::cout<<"\n[test_pumi] layer-wise pumi_ghost_createLayer (bd "<getPCU()->GetMPIComm()); + total_mcount_diff = m->getPCU()->Add(mcount_diff); if (!pumi_rank()) std::cout<<"\n[test_pumi] accumulative pumi_ghost_createLayer (bd "<Self()) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -89,9 +89,9 @@ void printDiagnostics(pcu::PCU *PCUObj) int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv,&PCUObj); @@ -101,5 +101,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/refine2x.cc b/test/refine2x.cc index c8e80b14d..1de67e672 100644 --- a/test/refine2x.cc +++ b/test/refine2x.cc @@ -83,9 +83,9 @@ class AnisotropicX: public ma::AnisotropicFunction { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (argc != 5) { if(0==PCUObj.Self()) @@ -130,5 +130,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/render.cc b/test/render.cc index 52ef4e3ab..91b92bedb 100644 --- a/test/render.cc +++ b/test/render.cc @@ -16,14 +16,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -60,6 +60,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/renderClass.cc b/test/renderClass.cc index 2d110c685..043ae61e8 100644 --- a/test/renderClass.cc +++ b/test/renderClass.cc @@ -32,16 +32,16 @@ static void number_dim(apf::Mesh* m, apf::FieldShape* shape, int dim, std::strin int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (!(argc == 4 || argc == 5)) { if ( !PCUObj.Self() ) { printf("Usage: %s \n", argv[0]); printf(" %s \n", argv[0]); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -76,7 +76,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/render_ascii.cc b/test/render_ascii.cc index 2d748f1ee..b6cc32300 100644 --- a/test/render_ascii.cc +++ b/test/render_ascii.cc @@ -13,14 +13,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -42,6 +42,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/reorder.cc b/test/reorder.cc index 94d6670bb..4e8878989 100644 --- a/test/reorder.cc +++ b/test/reorder.cc @@ -14,14 +14,14 @@ #include int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -46,5 +46,5 @@ int main(int argc, char** argv) { MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/repartition.cc b/test/repartition.cc index 07b85e6fd..78382ba1b 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -19,7 +19,7 @@ int inputPartCount = 1; struct CreateGroupCommResult{ bool isOriginal; - pcu::PCU *group_pcu_obj; + std::unique_ptr group_pcu_obj; }; void freeMesh(apf::Mesh* m) @@ -45,11 +45,9 @@ CreateGroupCommResult createGroupComm(pcu::PCU *PCUObj) no need to spend time computing a good contiguous number */ groupRank = 0; } - MPI_Comm groupComm; - MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); CreateGroupCommResult result; result.isOriginal = isOriginal; - result.group_pcu_obj = new pcu::PCU(groupComm); + result.group_pcu_obj = PCUObj->Split(group, groupRank); return result; } @@ -64,7 +62,7 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) "Unlike the [z]split tool, outPartCount does not have to be an integer\n" "multiple of inPartCount.\n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -96,9 +94,9 @@ void balance(apf::Mesh2* m) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU expanded_pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU expanded_pcu_obj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv,&expanded_pcu_obj); @@ -107,7 +105,7 @@ int main(int argc, char** argv) CreateGroupCommResult result = createGroupComm(&expanded_pcu_obj); if (result.isOriginal) - m = apf::loadMdsMesh(g, meshFile, result.group_pcu_obj); + m = apf::loadMdsMesh(g, meshFile, result.group_pcu_obj.get()); m = apf::expandMdsMesh(m, g, inputPartCount, &expanded_pcu_obj); balance(m); @@ -115,7 +113,7 @@ int main(int argc, char** argv) m->writeNative(outFile); freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/reposition.cc b/test/reposition.cc index c3d4b47ce..0cc574b57 100644 --- a/test/reposition.cc +++ b/test/reposition.cc @@ -7,9 +7,9 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #if 0 gmi_register_null(); @@ -44,5 +44,5 @@ int main(int argc, char** argv) ma::repositionVertex(m, v, 20, 1.0); apf::writeVtkFiles("after", m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc index a34934e5a..effb87900 100644 --- a/test/residualErrorEstimation_test.cc +++ b/test/residualErrorEstimation_test.cc @@ -25,9 +25,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -97,7 +97,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } void E_exact(const apf::Vector3 &x, apf::Vector3& E) diff --git a/test/rm_extrusion.cc b/test/rm_extrusion.cc index f415fd11f..701c7ab85 100644 --- a/test/rm_extrusion.cc +++ b/test/rm_extrusion.cc @@ -91,9 +91,9 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); @@ -145,5 +145,5 @@ int main(int argc, char** argv) if( should_log ) Sim_logOff(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc index 81896ad28..7f4e17e3d 100644 --- a/test/runSimxAnisoAdapt.cc +++ b/test/runSimxAnisoAdapt.cc @@ -74,9 +74,9 @@ apf::Mesh2* convertToPumi( const char* sizeName, const char* frameName, pcu::PCU *PCUObj); int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); MS_init(); // Call before calling Sim_readLicenseFile Sim_readLicenseFile(0); @@ -87,7 +87,7 @@ int main(int argc, char** argv) printf("USAGE: %s " " \n", argv[0]); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -229,7 +229,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); MS_exit(); } - MPI_Finalize(); + pcu::Finalize(); } void printModelStats(pGModel model) @@ -599,7 +599,7 @@ apf::Mesh2* convertToPumi( gmi_model* nullModel = gmi_load(".null"); apf::Mesh2* m2 = apf::makeEmptyMdsMesh(nullModel, dim, false, PCUObj); apf::GlobalToVert outMap; - apf::construct(m2, adaptedConns, adaptedNumElems, apf::Mesh::TET, outMap);; + apf::construct(m2, adaptedConns, adaptedNumElems, apf::Mesh::TET, outMap); apf::alignMdsRemotes(m2); apf::deriveMdsModel(m2); apf::setCoords(m2, adaptedCoords, adaptedNumVerts, outMap); diff --git a/test/scale.cc b/test/scale.cc index d321a066a..a59ce66c4 100644 --- a/test/scale.cc +++ b/test/scale.cc @@ -18,7 +18,7 @@ struct Scale { static void print_usage(char** argv, pcu::PCU *PCUObj) { if (! PCUObj->Self()) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -53,9 +53,9 @@ static void scale_mesh(apf::Mesh2* m, Scale const& s) { } int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); if (argc != 8) print_usage(argv, &PCUObj); @@ -75,5 +75,5 @@ int main(int argc, char** argv) { m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/serialize.cc b/test/serialize.cc index 74b7c5c47..080559509 100644 --- a/test/serialize.cc +++ b/test/serialize.cc @@ -25,14 +25,14 @@ struct GroupCode : public Parma_GroupCode int main( int argc, char* argv[]) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -59,6 +59,6 @@ int main( int argc, char* argv[]) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/shapefun.cc b/test/shapefun.cc index 7c02b1d2b..a6b38c394 100644 --- a/test/shapefun.cc +++ b/test/shapefun.cc @@ -242,9 +242,9 @@ void testPyramidVolume(pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_null(); testP1LineNodeValues(); @@ -262,5 +262,5 @@ int main(int argc, char** argv) testPrismVolume(&PCUObj); testPyramidVolume(&PCUObj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/shapefun2.cc b/test/shapefun2.cc index 5922c36e7..301b1692d 100644 --- a/test/shapefun2.cc +++ b/test/shapefun2.cc @@ -169,9 +169,9 @@ static void checkFieldShape(apf::FieldShape* fs, pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_null(); @@ -183,6 +183,6 @@ int main(int argc, char** argv) test::checkFieldShape(fs[i], &PCUObj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/simSpjToSmd.cc b/test/simSpjToSmd.cc index 1662c227c..aef78648e 100644 --- a/test/simSpjToSmd.cc +++ b/test/simSpjToSmd.cc @@ -70,7 +70,7 @@ int main(int argc, char* argv[]) // The meshing case doesn't seem to get fully deleted, so still want // to set the model so the smd file has the correct path AttCase_setModel(currentCase,model); - std::cout << "Deleting case "< int main(int argc, char** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (argc != 4) { if (PCUObj.Self() == 0) { @@ -24,7 +24,7 @@ int main(int argc, char** argv) { std::cout << "usage: smb2osh in.dmg in.smb out.osh\n"; std::cout << " or: smb2osh (usage)\n"; } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -39,5 +39,5 @@ int main(int argc, char** argv) { Omega_h::binary::write(argv[3], &om); } } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/smokeTesting.cmake b/test/smokeTesting.cmake index 854332587..38a3538a4 100644 --- a/test/smokeTesting.cmake +++ b/test/smokeTesting.cmake @@ -1,8 +1,18 @@ function(smoke_test TESTNAME PROCS EXE) set(tname smoke_test_${TESTNAME}) - add_test( - NAME ${tname} - COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN}) + if(SCOREC_NO_MPI) + if(PROCS EQUAL "1") + add_test(NAME ${tname} COMMAND ${VALGRIND} ${VALGRIND_ARGS} + ${EXE} ${ARGN}) + else() + return() + endif() + else() + add_test( + NAME ${tname} + COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} + ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN}) + endif() SET_TESTS_PROPERTIES(${tname} PROPERTIES LABELS "SMOKE_TEST" ) endfunction(smoke_test) diff --git a/test/snap.cc b/test/snap.cc index 13953fa2b..9b3807dc6 100644 --- a/test/snap.cc +++ b/test/snap.cc @@ -11,9 +11,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); MS_init(); SimModel_start(); @@ -31,6 +31,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/split.cc b/test/split.cc index 6dbcc4c18..6fd4d430b 100644 --- a/test/split.cc +++ b/test/split.cc @@ -37,22 +37,12 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) -{ - int self = PCUObj->Self(); - int groupRank = self / partitionFactor; - int group = self % partitionFactor; - MPI_Comm groupComm; - MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - return new pcu::PCU(groupComm); -} - void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -66,9 +56,9 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -84,15 +74,16 @@ int main(int argc, char** argv) g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - pcu::PCU *groupedPCUObj = getGroupedPCU(&PCUObj); + auto groupedPCUObj = PCUObj.Split( + PCUObj.Self() % partitionFactor, PCUObj.Self() / partitionFactor + ); if (isOriginal) { - m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); + m = apf::loadMdsMesh(g, meshFile, groupedPCUObj.get()); plan = getPlan(m); } //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh //on the globalPCU if(m != nullptr) m->switchPCU(&PCUObj); - delete groupedPCUObj; m = repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); @@ -104,6 +95,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/spr_test.cc b/test/spr_test.cc index 69276fdb2..bc4c28df7 100644 --- a/test/spr_test.cc +++ b/test/spr_test.cc @@ -17,9 +17,9 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; const char* outFile = argv[3]; const int order = atoi(argv[4]); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile, &PCUObj); @@ -37,6 +37,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/swapDoubles.cc b/test/swapDoubles.cc index 4b9e8a9f1..5570d6432 100644 --- a/test/swapDoubles.cc +++ b/test/swapDoubles.cc @@ -5,9 +5,9 @@ #include //cerr int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; const size_t n = 2; double *d_orig = new double[n]; std::iota(d_orig,d_orig+n,0); @@ -27,6 +27,6 @@ int main(int argc, char** argv) { delete [] d_orig; delete [] d; } - MPI_Finalize(); + pcu::Finalize(); return 0; } diff --git a/test/test_integrator.cc b/test/test_integrator.cc index 41f1cd320..853e8daf4 100644 --- a/test/test_integrator.cc +++ b/test/test_integrator.cc @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -22,9 +22,9 @@ class CountIntegrator : public apf::Integrator { } }; int main(int argc, char ** argv) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; // argument should be model, mesh PCU_ALWAYS_ASSERT(argc == 3); @@ -48,6 +48,6 @@ int main(int argc, char ** argv) { mesh->destroyNative(); apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); return 0; } diff --git a/test/test_matrix_grad.cc b/test/test_matrix_grad.cc index 80d702927..ea4667d5d 100644 --- a/test/test_matrix_grad.cc +++ b/test/test_matrix_grad.cc @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include @@ -92,9 +92,9 @@ int main(int argc, char* argv[]) std::cerr<<"Usage: "<destroyNative(); apf::destroyMesh(mesh); } - MPI_Finalize(); + pcu::Finalize(); return 0; } diff --git a/test/test_scaling.cc b/test/test_scaling.cc index f727a902b..c80d34fed 100644 --- a/test/test_scaling.cc +++ b/test/test_scaling.cc @@ -12,9 +12,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_register_mesh(); apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); @@ -27,5 +27,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/test_verify.cc b/test/test_verify.cc index 9802b09ad..e3209dcdf 100644 --- a/test/test_verify.cc +++ b/test/test_verify.cc @@ -10,9 +10,9 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); @@ -44,5 +44,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/testing.cmake b/test/testing.cmake index 77686ee6d..0a3b069f5 100644 --- a/test/testing.cmake +++ b/test/testing.cmake @@ -2,13 +2,41 @@ set(MESHES "" CACHE STRING "Extracted http://scorec.rpi.edu/pumi/pumi_test_meshes.tar.gz") function(mpi_test TESTNAME PROCS EXE) - add_test( - NAME ${TESTNAME} - COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN} - ) - set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS ${PROCS}) + if(SCOREC_NO_MPI) + if(${PROCS} EQUAL "1") + add_test( + NAME ${TESTNAME} + COMMAND ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN} + ) + endif() + else() + add_test( + NAME ${TESTNAME} + COMMAND ${MPIRUN} ${MPIRUN_PROCFLAG} ${PROCS} + ${VALGRIND} ${VALGRIND_ARGS} ${EXE} ${ARGN} + ) + set_tests_properties(${TESTNAME} PROPERTIES PROCESSORS ${PROCS}) + endif() endfunction(mpi_test) +function(mpi_test_depends) + cmake_parse_arguments(MPI_TEST_DEPENDS "" "" "TESTS;DEPENDS" ${ARGN}) + if (NOT DEFINED MPI_TEST_DEPENDS_TESTS OR NOT DEFINED MPI_TEST_DEPENDS_DEPENDS) + return() + endif() + if(SCOREC_NO_MPI) + # Check for test existence as it may be a multiproc test. + if(TEST "${TESTNAME}") + set_tests_properties(${MPI_TEST_DEPENDS_TESTS} PROPERTIES + DEPENDS "${MPI_TEST_DEPENDS_DEPENDS}") + endif() + else() + # Don't check if test exists because it's more likely a typo. + set_tests_properties(${MPI_TEST_DEPENDS_TESTS} PROPERTIES + DEPENDS "${MPI_TEST_DEPENDS_DEPENDS}") + endif() +endfunction() + mpi_test(shapefun 1 ./shapefun) mpi_test(shapefun2 1 ./shapefun2) mpi_test(bezierElevation 1 ./bezierElevation) @@ -135,7 +163,7 @@ if(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND) "${MDIR}/outmesh_4_parts.sms" 3504 WORKING_DIRECTORY ${MDIR}) - set_tests_properties(countBL_part_mesh PROPERTIES DEPENDS partition_sim) + mpi_test_depends(TESTS countBL_part_mesh DEPENDS partition_sim) endif() endif() endif(ENABLE_SIMMETRIX AND SIM_PARASOLID AND SIMMODSUITE_SimAdvMeshing_FOUND) @@ -266,7 +294,7 @@ mpi_test(inviscid_ghost 4 "${MDIR}/inviscid_egg.dmg" "${MDIR}/4/" "${MDIR}/vis") -set_tests_properties(inviscid_ghost PROPERTIES DEPENDS inviscid_ugrid) +mpi_test_depends(TESTS inviscid_ghost DEPENDS inviscid_ugrid) set(MDIR ${SMOKE_TEST_MESHES}/pipe) if(ENABLE_SIMMETRIX) @@ -395,7 +423,7 @@ else() "pipe_4_.smb" 2) endif() -set_tests_properties(split_4 PROPERTIES DEPENDS split_2) +mpi_test_depends(TESTS split_4 DEPENDS split_2) mpi_test(pipe_condense 4 ./serialize "${MDIR}/pipe.${GXT}" @@ -420,16 +448,15 @@ if(ENABLE_ZOLTAN) "${MDIR}/pipe.${GXT}" "pipe_4_.smb" "tet.smb") - set_tests_properties(ma_parallel tet_parallel - PROPERTIES DEPENDS split_4) + mpi_test_depends(TESTS ma_parallel DEPENDS split_4) + mpi_test_depends(TESTS tet_parallel DEPENDS split_4) endif() mpi_test(fieldReduce 4 ./fieldReduce "${MDIR}/pipe.${GXT}" "pipe_4_.smb") -set_tests_properties(pipe_condense verify_parallel fieldReduce verify_parallel - vtxElmMixedBalance - PROPERTIES DEPENDS split_4) +mpi_test_depends(TESTS pipe_condense verify_parallel fieldReduce + verify_parallel vtxElmMixedBalance DEPENDS split_4) set(MDIR ${MESHES}/torus) mpi_test(reorder 4 @@ -448,7 +475,7 @@ mpi_test(gap 4 "${MDIR}/torusBal4p/" "1.08" "${MDIR}/torusOpt4p/") -set_tests_properties(gap PROPERTIES DEPENDS balance) +mpi_test_depends(TESTS gap DEPENDS balance) mpi_test(applyMatrixFunc 1 ./applyMatrixFunc) if(ENABLE_ZOLTAN) @@ -696,8 +723,8 @@ mpi_test(test_verify 4 ./test_verify "${MDIR}/cube.dmg" "${MDIR}/pumi7k/4/cube.smb") -set_tests_properties(l2_shape_tet_parallel h1_shape_parallel test_verify - PROPERTIES DEPENDS "construct;constructThenGhost") +mpi_test_depends(TESTS l2_shape_tet_parallel h1_shape_parallel test_verify + DEPENDS "construct;constructThenGhost") set(MDIR ${MESHES}/nonmanifold) mpi_test(nonmanif_verify 1 ./verify @@ -713,7 +740,7 @@ mpi_test(nonmanif_verify2 2 ./verify "${MDIR}/nonmanifold.dmg" "nonmanifold_2_.smb") -set_tests_properties(nonmanif_verify2 PROPERTIES DEPENDS nonmanif_split2) +mpi_test_depends(TESTS nonmanif_verify2 DEPENDS nonmanif_split2) set(MDIR ${MESHES}/fusion) mpi_test(mkmodel_fusion 1 ./mkmodel @@ -774,7 +801,7 @@ if(ENABLE_SIMMETRIX) ./ma_test "${MDIR}/upright.smd" "67k/") - set_tests_properties(adapt_meshgen PROPERTIES DEPENDS parallel_meshgen) + mpi_test_depends(TESTS adapt_meshgen DEPENDS parallel_meshgen) endif() endif() if(SIM_PARASOLID) diff --git a/test/tetrahedronize.cc b/test/tetrahedronize.cc index ee0555833..ddcf2a6f8 100644 --- a/test/tetrahedronize.cc +++ b/test/tetrahedronize.cc @@ -14,9 +14,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -40,7 +40,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/torus_ma_test.cc b/test/torus_ma_test.cc index 5c0799848..4c76f19a2 100644 --- a/test/torus_ma_test.cc +++ b/test/torus_ma_test.cc @@ -46,9 +46,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); @@ -70,6 +70,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ugrid.cc b/test/ugrid.cc index ab35ff931..352ed246a 100644 --- a/test/ugrid.cc +++ b/test/ugrid.cc @@ -18,26 +18,16 @@ apf::Migration* getPlan(apf::Mesh* m, const int partitionFactor) return plan; } -pcu::PCU* getGroupedPCU(const int partitionFactor, pcu::PCU *PCUObj) -{ - int self = PCUObj->Self(); - int groupRank = self / partitionFactor; - int group = self % partitionFactor; - MPI_Comm groupComm; - MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - return new pcu::PCU(groupComm); -} - int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); @@ -47,9 +37,11 @@ int main(int argc, char** argv) gmi_model* g = gmi_load(".null"); apf::Mesh2* m = 0; apf::Migration* plan = 0; - pcu::PCU *groupedPCUObj = getGroupedPCU(partitionFactor, &PCUObj); + auto groupedPCUObj = PCUObj.Split( + PCUObj.Self() % partitionFactor, PCUObj.Self() / partitionFactor + ); if (isOriginal) { - m = apf::loadMdsFromUgrid(g, argv[1], groupedPCUObj); + m = apf::loadMdsFromUgrid(g, argv[1], groupedPCUObj.get()); apf::deriveMdsModel(m); m->verify(); plan = getPlan(m, partitionFactor); @@ -57,7 +49,6 @@ int main(int argc, char** argv) //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh //on the globalPCU if(m != nullptr) m->switchPCU(&PCUObj); - delete groupedPCUObj; m = repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); gmi_write_dmg(g,argv[2]); @@ -65,6 +56,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/ugridptnstats.cc b/test/ugridptnstats.cc index c9a67dac4..a488aeb48 100644 --- a/test/ugridptnstats.cc +++ b/test/ugridptnstats.cc @@ -18,9 +18,9 @@ const double weights[8] = {vtxw, edgew, triw, quadw, tetw, hexw, przw, pyrw}; int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_null(); PCU_ALWAYS_ASSERT( 3 == argc ); @@ -30,5 +30,5 @@ int main(int argc, char** argv) gmi_model* g = gmi_load(".null"); apf::printUgridPtnStats(g,ugridfile,ptnfile,weights,&PCUObj); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/uniform.cc b/test/uniform.cc index 2d45b167b..9ca428f12 100644 --- a/test/uniform.cc +++ b/test/uniform.cc @@ -21,7 +21,7 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) if ( argc != 4 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -32,9 +32,9 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -63,6 +63,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/verify.cc b/test/verify.cc index 76f94e055..1e12067c2 100644 --- a/test/verify.cc +++ b/test/verify.cc @@ -14,9 +14,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -37,7 +37,7 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/verify_2nd_order_shapes.cc b/test/verify_2nd_order_shapes.cc index acfc64f48..4afde62e6 100644 --- a/test/verify_2nd_order_shapes.cc +++ b/test/verify_2nd_order_shapes.cc @@ -16,14 +16,14 @@ #include int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 2 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -111,5 +111,5 @@ int main(int argc, char** argv) { MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/verify_convert.cc b/test/verify_convert.cc index f312d202b..3f268cb83 100644 --- a/test/verify_convert.cc +++ b/test/verify_convert.cc @@ -42,9 +42,9 @@ class twox : public apf::Function { }; int main(int argc, char* argv[]) { - MPI_Init(&argc, &argv); + pcu::Init(&argc, &argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_null(); @@ -162,6 +162,6 @@ int main(int argc, char* argv[]) apf::destroyMesh(m2); apf::destroyMesh(m3); } - MPI_Finalize(); + pcu::Finalize(); return 0; } diff --git a/test/visualizeAnisoSizes.cc b/test/visualizeAnisoSizes.cc index f27b25b7c..6f7ae0be3 100644 --- a/test/visualizeAnisoSizes.cc +++ b/test/visualizeAnisoSizes.cc @@ -42,9 +42,9 @@ void visualizeSizeField( int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if (argc < 8) { if (PCUObj.Self() == 0) { @@ -53,7 +53,7 @@ int main(int argc, char** argv) printf("USAGE2: %s " " \n", argv[0]); } - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } @@ -88,7 +88,7 @@ int main(int argc, char** argv) #endif } - MPI_Finalize(); + pcu::Finalize(); } void safe_mkdir(const char* path) diff --git a/test/viz.cc b/test/viz.cc index cf99e58fa..58a122b5e 100644 --- a/test/viz.cc +++ b/test/viz.cc @@ -34,10 +34,10 @@ namespace { int main(int argc, char** argv) { int provided; - MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided); + pcu::Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided); PCU_ALWAYS_ASSERT(provided==MPI_THREAD_MULTIPLE); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); @@ -106,5 +106,5 @@ int main(int argc, char** argv) freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/vtxBalance.cc b/test/vtxBalance.cc index 163330f39..bfa9ab570 100644 --- a/test/vtxBalance.cc +++ b/test/vtxBalance.cc @@ -29,14 +29,14 @@ namespace { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -69,5 +69,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/vtxEdgeElmBalance.cc b/test/vtxEdgeElmBalance.cc index b79239277..07421ce91 100644 --- a/test/vtxEdgeElmBalance.cc +++ b/test/vtxEdgeElmBalance.cc @@ -44,14 +44,14 @@ namespace { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 6); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 6 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -87,5 +87,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/vtxElmBalance.cc b/test/vtxElmBalance.cc index 12a47e39c..74a4071cd 100644 --- a/test/vtxElmBalance.cc +++ b/test/vtxElmBalance.cc @@ -33,14 +33,14 @@ namespace { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -60,5 +60,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/vtxElmMixedBalance.cc b/test/vtxElmMixedBalance.cc index 3aa55fe99..53576b323 100644 --- a/test/vtxElmMixedBalance.cc +++ b/test/vtxElmMixedBalance.cc @@ -16,9 +16,9 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(argc==3); const char* modelFile = argv[1]; const char* meshFile = argv[2]; - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -42,6 +42,6 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/writeIPFieldTest.cc b/test/writeIPFieldTest.cc index 8a0f60e01..b4de654b3 100644 --- a/test/writeIPFieldTest.cc +++ b/test/writeIPFieldTest.cc @@ -8,9 +8,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); gmi_register_mesh(); apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); @@ -20,5 +20,5 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/writePart.cc b/test/writePart.cc index 4d631ec0e..a9320015d 100644 --- a/test/writePart.cc +++ b/test/writePart.cc @@ -7,14 +7,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU pcu_obj; lion_set_verbosity(1); if ( argc != 5 ) { if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); @@ -24,7 +24,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/writeVtxPtn.cc b/test/writeVtxPtn.cc index 600519842..25eb9e27d 100644 --- a/test/writeVtxPtn.cc +++ b/test/writeVtxPtn.cc @@ -22,7 +22,7 @@ namespace { if ( argc != 4 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -32,9 +32,9 @@ namespace { int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv,&PCUObj); @@ -42,5 +42,5 @@ int main(int argc, char** argv) Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/xgc_split.cc b/test/xgc_split.cc index c359c7447..0fd6d8c8e 100644 --- a/test/xgc_split.cc +++ b/test/xgc_split.cc @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -19,7 +18,7 @@ void getConfig(int argc, char** argv, pcu::PCU* PCUObj) if (argc < 4) { if (!PCUObj->Self()) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -54,9 +53,9 @@ Migration* get_xgc_plan(pGeom g, pMesh m) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; pumi_load_pcu(&PCUObj); getConfig(argc,argv,&PCUObj); @@ -95,6 +94,6 @@ int main(int argc, char** argv) pumi_mesh_delete(m); } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/zbalance.cc b/test/zbalance.cc index 7409e5291..163c29f8d 100644 --- a/test/zbalance.cc +++ b/test/zbalance.cc @@ -18,14 +18,14 @@ int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); if ( argc != 4 ) { if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } #ifdef HAVE_SIMMETRIX @@ -57,5 +57,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/test/zsplit.cc b/test/zsplit.cc index 547201a9a..1b2bc5455 100644 --- a/test/zsplit.cc +++ b/test/zsplit.cc @@ -39,22 +39,12 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) -{ - int self = PCUObj->Self(); - int groupRank = self / partitionFactor; - int group = self % partitionFactor; - MPI_Comm groupComm; - MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - return new pcu::PCU(groupComm); -} - void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); - MPI_Finalize(); + pcu::Finalize(); exit(EXIT_FAILURE); } modelFile = argv[1]; @@ -68,9 +58,9 @@ void getConfig(int argc, char** argv, pcu::PCU *PCUObj) int main(int argc, char** argv) { - MPI_Init(&argc,&argv); + pcu::Init(&argc,&argv); { - pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::PCU PCUObj; lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -86,15 +76,16 @@ int main(int argc, char** argv) g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - pcu::PCU *groupedPCUObj = getGroupedPCU(&PCUObj); + auto groupedPCUObj = PCUObj.Split( + PCUObj.Self() % partitionFactor, PCUObj.Self() / partitionFactor + ); if (isOriginal) { - m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); + m = apf::loadMdsMesh(g, meshFile, groupedPCUObj.get()); plan = getPlan(m); } //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh //on the globalPCU if(m != nullptr) m->switchPCU(&PCUObj); - delete groupedPCUObj; m = apf::repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); @@ -106,5 +97,5 @@ int main(int argc, char** argv) MS_exit(); #endif } - MPI_Finalize(); + pcu::Finalize(); } diff --git a/zoltan/CMakeLists.txt b/zoltan/CMakeLists.txt index 0901f9df5..92e9638a2 100644 --- a/zoltan/CMakeLists.txt +++ b/zoltan/CMakeLists.txt @@ -9,6 +9,10 @@ xsdk_add_tpl(ZOLTAN) xsdk_add_tpl(PARMETIS) message(STATUS "ENABLE_ZOLTAN: " ${ENABLE_ZOLTAN}) +if(SCOREC_NO_MPI AND ENABLE_ZOLTAN) + message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with Zoltan.") +endif() + # Let CMake know where to find custom find package implementations set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} @@ -47,7 +51,6 @@ add_library(apf_zoltan ${SOURCES}) # Include directories target_include_directories(apf_zoltan INTERFACE $ - $ ) # Link this package to these libraries diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index 63bed1064..7fe724fc5 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -271,9 +271,12 @@ ZoltanData::ZoltanData(ZoltanMesh* zb_) : zb(zb_) MPI_Comm comm; if (zb->isLocal) comm = MPI_COMM_SELF; - else - comm = zb->mesh->getPCU()->GetMPIComm(); + else { + zb->mesh->getPCU()->DupComm(&comm); + } ztn =Zoltan_Create(comm); + if (!zb->isLocal) + MPI_Comm_free(&comm); // Zoltan duplicates too, so free our reference. import_gids = NULL; import_lids = NULL; import_procs = NULL;