Add threads framework

Add a framework to support different types of threading models including
user space thread packages such as Qthreads and argobot:

https://github.com/pmodels/argobots

https://github.com/Qthreads/qthreads

The default threading model is pthreads.  Alternate thread models are
specificed at configure time using the --with-threads=X option.

The framework is static.  The theading model to use is selected at
Open MPI configure/build time.

mca/threads: implement Argobots threading layer

config: fix thread configury

- Add double quotations
- Change Argobot to Argobots
config: implement Argobots check

If the poll time is too long, MPI hangs.

This quick fix just sets it to 0, but it is not good for the
Pthreads version. Need to find a good way to abstract it.

Note that even 1 (= 1 millisecond) causes disastrous performance
degradation.

rework threads MCA framework configury

It now works more like the ompi/mca/rte configury,
modulo some edge items that are special for threading package
linking, etc.

qthreads module
some argobots cleanup

Signed-off-by: Noah Evans <noah.evans@gmail.com>
Signed-off-by: Shintaro Iwasaki <siwasaki@anl.gov>
Signed-off-by: Howard Pritchard <howardp@lanl.gov>

Author: npe9

config/opal_check_pthread_pids.m4

@@ -26,6 +26,8 @@
 # Copyright (c) 2018      Amazon.com, Inc. or its affiliates.
 #                         All Rights reserved.
 # Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
+# Copyright (c) 2019      Triad National Security, LLC. All rights
+#                         reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -945,22 +947,6 @@ OPAL_CHECK_BROKEN_QSORT
 # all: type of getsockopt optlen
 # all: type of recvfrom optlen
 
-#
-# Check out what thread support we have
-#
-OPAL_CONFIG_THREADS
-
-CFLAGS="$CFLAGS $THREAD_CFLAGS"
-CPPFLAGS="$CPPFLAGS $THREAD_CPPFLAGS"
-LDFLAGS="$LDFLAGS $THREAD_LDFLAGS"
-LIBS="$LIBS $THREAD_LIBS"
-
-OPAL_WRAPPER_FLAGS_ADD([CFLAGS], [$THREAD_CFLAGS])
-OPAL_WRAPPER_FLAGS_ADD([CXXFLAGS], [$THREAD_CXXFLAGS])
-OPAL_WRAPPER_FLAGS_ADD([FCFLAGS], [$THREAD_FCFLAGS])
-OPAL_WRAPPER_FLAGS_ADD([LDFLAGS], [$THREAD_LDFLAGS])
-# no need to update WRAPPER_EXTRA_LIBS - we'll get it from LT later
-
 #
 # What is the local equivalent of "ln -s"
 #
@@ -1113,6 +1099,14 @@ AC_MSG_RESULT([$opal_subdir_args])
 
 OPAL_MCA
 
+#
+# Now that we know how to support threads with wrappers, update
+#
+OPAL_WRAPPER_FLAGS_ADD([CFLAGS], [$THREAD_CFLAGS])
+OPAL_WRAPPER_FLAGS_ADD([CXXFLAGS], [$THREAD_CXXFLAGS])
+OPAL_WRAPPER_FLAGS_ADD([FCFLAGS], [$THREAD_FCFLAGS])
+OPAL_WRAPPER_FLAGS_ADD([LDFLAGS], [$THREAD_LDFLAGS])
+
 m4_ifdef([project_ompi], [OMPI_REQUIRE_ENDPOINT_TAG_FINI])
 
 # Last minute disable of OpenSHMEM if we didn't find any oshmem SPMLs

config/opal_config_threads.m4

@@ -232,7 +232,7 @@
 #include "ompi_config.h"
 
 #include "opal/class/opal_bitmap.h"
-#include "opal/threads/mutex.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/sys/atomic.h"
 
 #include "ompi/attribute/attribute.h"

configure.ac

@@ -41,7 +41,7 @@
 #include "opal/util/string_copy.h"
 
 #include "ompi/proc/proc.h"
-#include "opal/threads/mutex.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/util/bit_ops.h"
 #include "opal/util/output.h"
 #include "ompi/mca/topo/topo.h"

ompi/attribute/attribute.c

@@ -36,7 +36,7 @@
 #include "opal/class/opal_hash_table.h"
 #include "opal/util/info_subscriber.h"
 #include "ompi/errhandler/errhandler.h"
-#include "opal/threads/mutex.h"
+#include "opal/mca/threads/mutex.h"
 #include "ompi/communicator/comm_request.h"
 
 #include "mpi.h"

ompi/communicator/comm.c

@@ -30,7 +30,7 @@
 #include "mpi.h"
 #include "opal/class/opal_list.h"
 #include "ompi/errhandler/errhandler.h"
-#include "opal/threads/mutex.h"
+#include "opal/mca/threads/mutex.h"
 #include "opal/util/info_subscriber.h"
 #include "ompi/mca/io/io.h"
 

ompi/communicator/communicator.h

@@ -32,7 +32,7 @@
 #include "opal/util/info.h"
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_pointer_array.h"
-#include "opal/threads/mutex.h"
+#include "opal/mca/threads/mutex.h"
 
 #include "opal/mca/base/mca_base_var_enum.h"
 

ompi/file/file.h

@@ -58,7 +58,7 @@ writing 4x4 matrix to monitoring_avg.mat
 #define OMPI_COMPILING_FORTRAN_WRAPPERS 1
 #endif
 
-#include "opal/threads/thread_usage.h"
+#include "opal/mca/threads/thread_usage.h"
 
 #include "ompi/include/mpi.h"
 #include "ompi/mpi/fortran/base/constants.h"

ompi/info/info.h

@@ -29,7 +29,7 @@
 #include "mpi.h"
 #include "opal/class/opal_list.h"
 #include "ompi/errhandler/errhandler.h"
-#include "opal/threads/mutex.h"
+#include "opal/mca/threads/mutex.h"
 #include "ompi/file/file.h"
 #include "ompi/mca/io/io.h"
 #include "ompi/mca/fs/fs.h"