From 7e6fd4fc9a4fb2c1809110acefe9fed6e57b0495 Mon Sep 17 00:00:00 2001 From: Andre Vehreschild Date: Tue, 8 Oct 2024 14:07:24 +0200 Subject: [PATCH 1/6] Fix whitespace/formatting/typos. --- src/CMakeLists.txt | 4 ++-- src/tests/utilities/opencoarrays_object_interface.f90 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 22587b03f..091756eb7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,5 @@ -add_subdirectory(application-binary-interface ) -add_subdirectory(runtime-libraries ) +add_subdirectory(application-binary-interface) +add_subdirectory(runtime-libraries) add_subdirectory(tests) set(N_CPU ${N_CPU} PARENT_SCOPE) diff --git a/src/tests/utilities/opencoarrays_object_interface.f90 b/src/tests/utilities/opencoarrays_object_interface.f90 index b1019b5fd..2b5458cbe 100644 --- a/src/tests/utilities/opencoarrays_object_interface.f90 +++ b/src/tests/utilities/opencoarrays_object_interface.f90 @@ -1,6 +1,6 @@ module opencoarrays_object_interface - !! Object pattern abstract type to provide a universal interface to a userd-defined derived type ouptput - !! capability specified in a generic binding + !! Object pattern abstract type to provide a universal interface to a user-defined + !! derived type output capability specified in a generic binding implicit none type, abstract :: object From 53f4d2c607f81653c8da163287e5445080fc89ff Mon Sep 17 00:00:00 2001 From: Andre Vehreschild Date: Tue, 8 Oct 2024 14:09:11 +0200 Subject: [PATCH 2/6] Fix object dependencies of build artifacts. --- .../pde_solvers/coarrayBurgers/CMakeLists.txt | 24 ++++++++++++++----- .../coarrayHeatSimplified/CMakeLists.txt | 16 +++++++------ src/tests/utilities/CMakeLists.txt | 22 +++++++---------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/tests/integration/pde_solvers/coarrayBurgers/CMakeLists.txt b/src/tests/integration/pde_solvers/coarrayBurgers/CMakeLists.txt index bd83ed655..22401e9d1 100644 --- a/src/tests/integration/pde_solvers/coarrayBurgers/CMakeLists.txt +++ b/src/tests/integration/pde_solvers/coarrayBurgers/CMakeLists.txt @@ -19,13 +19,25 @@ else() endif() set(CMAKE_Fortran_COMPILER "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf") -add_executable(coarray_burgers_pde - main.F90 - global_field.F90 - local_field.F90 +add_library(cbp_base STATIC + ${config_directory}/compiler_capabilities.txt ${library_directory}/ForTrilinos_assertion_utility.F90 ${library_directory}/object_interface.F90 ${library_directory}/co_object_interface.F90 ) -add_dependencies(coarray_burgers_pde caf_mpi_static) -target_include_directories(coarray_burgers_pde PRIVATE ${config_directory}) +add_library(cbp_local STATIC + ${config_directory}/compiler_capabilities.txt + local_field.F90 +) +add_executable(coarray_burgers_pde + ${config_directory}/compiler_capabilities.txt + global_field.F90 + main.F90 +) +target_link_libraries(cbp_local PUBLIC cbp_base caf_mpi_static) +target_link_libraries(coarray_burgers_pde PRIVATE cbp_local) +set_target_properties(cbp_base cbp_local coarray_burgers_pde + PROPERTIES + Fortran_MODULE_DIRECTORY "${config_directory}" + INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR};${config_directory}" +) diff --git a/src/tests/integration/pde_solvers/coarrayHeatSimplified/CMakeLists.txt b/src/tests/integration/pde_solvers/coarrayHeatSimplified/CMakeLists.txt index 84ec5e2cc..ee3a4e37c 100644 --- a/src/tests/integration/pde_solvers/coarrayHeatSimplified/CMakeLists.txt +++ b/src/tests/integration/pde_solvers/coarrayHeatSimplified/CMakeLists.txt @@ -1,11 +1,13 @@ set(CMAKE_Fortran_COMPILER "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/caf") -add_library(local_field OBJECT local_field.f90) -add_library(global_field OBJECT global_field.f90) -add_dependencies(local_field caf_mpi_static) -add_dependencies(global_field local_field caf_mpi_static) +add_library(local_field STATIC local_field.f90) +add_library(global_field STATIC global_field.f90) add_executable(co_heat main.f90 - $ - $ ) -add_dependencies(co_heat caf_mpi_static) +target_link_libraries(global_field PUBLIC local_field caf_mpi_static) +target_link_libraries(co_heat PRIVATE global_field) +set_target_properties(local_field global_field co_heat + PROPERTIES + INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}" + Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}" +) diff --git a/src/tests/utilities/CMakeLists.txt b/src/tests/utilities/CMakeLists.txt index 9def3e858..de7d91b7d 100644 --- a/src/tests/utilities/CMakeLists.txt +++ b/src/tests/utilities/CMakeLists.txt @@ -31,27 +31,21 @@ if ("${CMAKE_Fortran_COMPILER_ID}" MATCHES "GNU" ) set(CMAKE_REQUIRED_FLAGS ${OLD_REQUIRED_FLAGS}) endif() -add_library( oc_test_interfaces OBJECT +add_library( oc_test_interfaces STATIC opencoarrays_object_interface.f90 oc_assertions_interface.F90 ) add_library( opencoarrays_test_utilities STATIC - oc_assertions_implementation.F90 $ + oc_assertions_implementation.F90 ) -set_target_properties(opencoarrays_test_utilities - PROPERTIES - Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}") -set_target_properties(oc_test_interfaces +target_link_libraries(opencoarrays_test_utilities PUBLIC oc_test_interfaces) +set_target_properties(oc_test_interfaces opencoarrays_test_utilities PROPERTIES - Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}") - -target_compile_options(oc_test_interfaces - PUBLIC - "-fcoarray=lib") -target_compile_options(opencoarrays_test_utilities - PUBLIC - "-fcoarray=lib") + Fortran_MODULE_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}" + INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${mod_dir_tail}" + COMPILE_OPTIONS "-fcoarray=lib" + ) if(HAVE_ERROR_STOP_IN_PURE) target_compile_definitions(oc_test_interfaces From 5fc8ca9dd309965a7ba21582f7010cafce58f481 Mon Sep 17 00:00:00 2001 From: Andre Vehreschild Date: Tue, 8 Oct 2024 14:09:58 +0200 Subject: [PATCH 3/6] Prevent caf script to check for caf_mpi_static when -cpp is specified. --- src/script-templates/caf.in | 41 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/script-templates/caf.in b/src/script-templates/caf.in index be64c8275..ab0e5f8a9 100755 --- a/src/script-templates/caf.in +++ b/src/script-templates/caf.in @@ -161,14 +161,13 @@ __caf_err_report() { trap '__caf_err_report "${FUNCNAME:-.}" ${LINENO}' ERR # See if we are compiling or compiling and/or linking -__only_compiling () { - for arg in "${@}"; do - if [[ "${arg}" == "-c" ]]; then - return 0 - fi - done - return 1 -} +with_link="true" +for arg in "${@}"; do + if [[ "${arg}" == "-c" || "${arg}" == "-cpp" ]]; then + with_link="false" + break + fi +done #-------------------------------------------------------------------------- # End configured variables, now process them and build compile/link command @@ -223,16 +222,14 @@ if [[ -n "${mpi_compile_flags[*]:-}" ]]; then caf_pre_flags+=("${compileflag}") done fi -if [[ -n "${mpi_link_flags[*]:-}" ]]; then - if ! __only_compiling "${@}"; then - for linkflag in "${mpi_link_flags[@]:-}"; do - caf_pre_flags+=("${linkflag}") - done - fi +if [[ -n "${mpi_link_flags[*]:-}" && "${with_link}" == "true" ]]; then + for linkflag in "${mpi_link_flags[@]:-}"; do + caf_pre_flags+=("${linkflag}") + done fi # Now do libraries, IN CORRECT ORDER, to append to command -if [[ -n "${caf_libs[*]:-}" ]]; then +if [[ "${with_link}" == "true" && -n "${caf_libs[*]:-}" ]]; then for lib in "${caf_libs[@]:-}"; do caf_added_libs+=("$(substitute_lib "${prefix%/}/${lib}")") done @@ -242,7 +239,7 @@ if [[ -n "${threads_lib}" ]]; then caf_added_libs+=("${threads_lib}") fi -if [[ -n "${mpi_libs[*]:-}" ]]; then +if [[ "${with_link}" == "true" && -n "${mpi_libs[*]:-}" ]]; then for lib in "${mpi_libs[@]:-}"; do caf_added_libs+=("$(substitute_lib "${lib}")") done @@ -330,10 +327,8 @@ elif [[ ${1} == '-s' || ${1} == '--show' || ${1} == '-show' ]]; then if [[ "${args}" ]]; then compiler_args+=("${args}") fi - if [[ "${caf_added_libs[*]:-}" ]]; then - if ! __only_compiling "${@}"; then - compiler_args+=("${caf_added_libs[@]}") - fi + if [[ "${caf_added_libs[*]:-}" && "${with_link}" == "true" ]]; then + compiler_args+=("${caf_added_libs[@]}") fi echo "${cafc}" "${compiler_args[@]}" exit 0 @@ -365,10 +360,8 @@ fi if [[ "${*:-}" ]]; then compiler_args+=("${@}") fi -if [[ "${caf_added_libs[*]:-}" ]]; then - if ! __only_compiling "${@}" ; then - compiler_args+=("${caf_added_libs[@]}") - fi +if [[ "${caf_added_libs[*]:-}" && "${with_link}" == "true" ]]; then + compiler_args+=("${caf_added_libs[@]}") fi #set -o xtrace # Show what we're doing set +o errtrace From 2351334df6404c5d4f963a1efb3aff49fe26f078 Mon Sep 17 00:00:00 2001 From: Andre Vehreschild Date: Tue, 8 Oct 2024 14:10:39 +0200 Subject: [PATCH 4/6] Rename modules to make 100% sure they are not mixed up with the ones from the burgers example. --- .../pde_solvers/coarrayHeatSimplified/global_field.f90 | 4 ++-- .../pde_solvers/coarrayHeatSimplified/local_field.f90 | 2 +- .../integration/pde_solvers/coarrayHeatSimplified/main.f90 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tests/integration/pde_solvers/coarrayHeatSimplified/global_field.f90 b/src/tests/integration/pde_solvers/coarrayHeatSimplified/global_field.f90 index 037471c53..dcbae2447 100644 --- a/src/tests/integration/pde_solvers/coarrayHeatSimplified/global_field.f90 +++ b/src/tests/integration/pde_solvers/coarrayHeatSimplified/global_field.f90 @@ -24,8 +24,8 @@ ! ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ! (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ! -module global_field_module - use local_field_module, only : local_field +module chs_global_field_module + use chs_local_field_module, only : local_field implicit none private public :: global_field diff --git a/src/tests/integration/pde_solvers/coarrayHeatSimplified/local_field.f90 b/src/tests/integration/pde_solvers/coarrayHeatSimplified/local_field.f90 index 204f73a2d..e77d9aa59 100644 --- a/src/tests/integration/pde_solvers/coarrayHeatSimplified/local_field.f90 +++ b/src/tests/integration/pde_solvers/coarrayHeatSimplified/local_field.f90 @@ -24,7 +24,7 @@ ! ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ! (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ! -module local_field_module +module chs_local_field_module implicit none private public :: local_field diff --git a/src/tests/integration/pde_solvers/coarrayHeatSimplified/main.f90 b/src/tests/integration/pde_solvers/coarrayHeatSimplified/main.f90 index ee77ab6d6..c9e47f5e2 100644 --- a/src/tests/integration/pde_solvers/coarrayHeatSimplified/main.f90 +++ b/src/tests/integration/pde_solvers/coarrayHeatSimplified/main.f90 @@ -27,7 +27,7 @@ program main use IEEE_arithmetic, only : IEEE_is_NaN - use global_field_module, only : global_field + use chs_global_field_module, only : global_field implicit none type(global_field) :: T,laplacian_T,T_half real, parameter :: alpha=1.,dt=0.0001,final_time=1.,tolerance=1.E-3 From 5491714d7fdfcc50dfe23f895ce77c7abd7aa1ba Mon Sep 17 00:00:00 2001 From: Andre Vehreschild Date: Wed, 2 Oct 2024 15:35:50 +0200 Subject: [PATCH 5/6] Purge communicator before free on mpich4+. Mpich from 4.0 on seems to bug when there is a message dangling on freeing the communicator. --- src/runtime-libraries/mpi/CMakeLists.txt | 4 ++++ src/runtime-libraries/mpi/mpi_caf.c | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/runtime-libraries/mpi/CMakeLists.txt b/src/runtime-libraries/mpi/CMakeLists.txt index b79334c11..6341cdf73 100644 --- a/src/runtime-libraries/mpi/CMakeLists.txt +++ b/src/runtime-libraries/mpi/CMakeLists.txt @@ -154,6 +154,10 @@ if (mpi_version_out MATCHES "[Oo]pen[ -][Mm][Pp][Ii]") if(NOT DEFINED ENV{TRAVIS}) message( STATUS "Open-MPI back end detected, passing --allow-run-as-root to allow tests to pass when run with sudo or as root." ) endif() +elseif (mpi_version_out MATCHES "HYDRA") + message(STATUS "MPICH detected") + target_compile_definitions(caf_mpi PRIVATE MPI_CLEAR_COMM_BEFORE_FREE) + target_compile_definitions(caf_mpi_static PRIVATE MPI_CLEAR_COMM_BEFORE_FREE) endif () if("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") diff --git a/src/runtime-libraries/mpi/mpi_caf.c b/src/runtime-libraries/mpi/mpi_caf.c index ce63f7244..5e5d71d77 100644 --- a/src/runtime-libraries/mpi/mpi_caf.c +++ b/src/runtime-libraries/mpi/mpi_caf.c @@ -1104,6 +1104,24 @@ finalize_internal(int status_code) ierr = MPI_Finalize(); chk_err(ierr); } #else +#ifdef MPI_CLEAR_COMM_BEFORE_FREE + { + int probe_flag; + MPI_Status status; + do { + ierr = MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, CAF_COMM_WORLD, &probe_flag, + &status); /* error is not of interest. */ + if (probe_flag) { + int cnt; + MPI_Get_count(&status, MPI_BYTE, &cnt); + void * buf = alloca(cnt); + ierr = MPI_Recv(buf, cnt, MPI_BYTE, status.MPI_SOURCE, status.MPI_TAG, + CAF_COMM_WORLD, &status); chk_err(ierr); + } + } while (probe_flag); + } +#endif + dprint("freeing caf's communicator.\n"); ierr = MPI_Comm_free(&CAF_COMM_WORLD); chk_err(ierr); CAF_Win_unlock_all(*stat_tok); @@ -1112,6 +1130,7 @@ finalize_internal(int status_code) /* Only call Finalize if CAF runtime Initialized MPI. */ if (caf_owns_mpi) { + dprint("Finalizing MPI.\n"); ierr = MPI_Finalize(); chk_err(ierr); } #endif From 2bae1150a6b593de70c69f13dbda98b5b6e01943 Mon Sep 17 00:00:00 2001 From: Andre Vehreschild Date: Wed, 9 Oct 2024 12:20:51 +0200 Subject: [PATCH 6/6] Use a global property for indicating openmpi. Using just a variable to indicate openmpi did not work reliably with all generators, i.e. at least with ninja the variable was not set in the top-level CMakeLists.txt and therefore additional options were not set leading to test failures. --- CMakeLists.txt | 6 ++++++ src/runtime-libraries/mpi/CMakeLists.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ea5fc68b..47fbf4b87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -644,6 +644,11 @@ define_property(TARGET FULL_DOCS "Property to mark executable targets run as tests that they require 2^n images." ) +#------------------------------------------------------------------------------------- +# Add global openmpi property, because using a variable an setting in parent scope did +# not work as expected, i.e., not at all, on Linux Fedora 39. +#------------------------------------------------------------------------------------- +define_property(GLOBAL PROPERTY openmpi BRIEF_DOCS "True when mpi is openMPI.") #------------------------------- # Recurse into the src directory @@ -715,6 +720,7 @@ function(add_caf_test name num_caf_img test_target) endif() endif() # Add a host file for OMPI + get_property(openmpi GLOBAL PROPERTY openmpi) if ( openmpi ) set(test_parameters --hostfile ${CMAKE_BINARY_DIR}/hostfile) endif() diff --git a/src/runtime-libraries/mpi/CMakeLists.txt b/src/runtime-libraries/mpi/CMakeLists.txt index 6341cdf73..8155b17b5 100644 --- a/src/runtime-libraries/mpi/CMakeLists.txt +++ b/src/runtime-libraries/mpi/CMakeLists.txt @@ -147,7 +147,7 @@ execute_process(COMMAND ${MPIEXEC_EXECUTABLE} --version OUTPUT_VARIABLE mpi_version_out) if (mpi_version_out MATCHES "[Oo]pen[ -][Mm][Pp][Ii]") message( STATUS "OpenMPI detected") - set ( openmpi true PARENT_SCOPE) + set_property(GLOBAL PROPERTY openmpi true) # Write out a host file because OMPI's mpiexec is dumb file(WRITE ${CMAKE_BINARY_DIR}/hostfile "${HOST_NAME} slots=${N_CPU}\n") message( STATUS "hostfile written to: ${CMAKE_BINARY_DIR}/hostfile")