Update hpctests to obey UCX_NET_DEVICES when RoCE devices present (#421)

* Turn off higher priority MPI net devices

* Update pingmatrix.sh.j2

* Update pingmatrix.sh.j2

* Update pingpong.sh.j2

* Replace j2 comments with bash

* Update pingpong.sh.j2

---------

Co-authored-by: Steve Brasier <33413598+sjpb@users.noreply.github.com>

Author: bertiethorpe

ansible/roles/hpctests/templates/pingmatrix.sh.j2

@@ -16,4 +16,8 @@ echo UCX_NET_DEVICES: $UCX_NET_DEVICES
 module load {{ hpctests_pingmatrix_modules | join(' ' ) }}
 
 mpicc -o nxnlatbw mpi_nxnlatbw.c
-mpirun nxnlatbw
+
+# mpirun flags force using UCX TCP transports, overriding higher
+# priority of OpenMPI btl/openib component, which is also using RDMA
+# https://wiki.stackhpc.com/s/985dae84-7bd8-4924-94b7-9629a7827100 
+mpirun -mca pml_ucx_tls any -mca pml_ucx_devices any nxnlatbw

ansible/roles/hpctests/templates/pingpong.sh.j2

@@ -16,4 +16,8 @@ echo UCX_NET_DEVICES: $UCX_NET_DEVICES
 module load {{ hpctests_pingpong_modules | join(' ' ) }}
 
 #srun --mpi=pmi2 IMB-MPI1 pingpong # doesn't work in ohpc v2.1
-mpirun IMB-MPI1 pingpong
+
+# mpirun flags force using UCX TCP transports, overriding higher 
+# priority of OpenMPI btl/openib component, which is also using RDMA 
+# https://wiki.stackhpc.com/s/985dae84-7bd8-4924-94b7-9629a7827100
+mpirun -mca pml_ucx_tls any -mca pml_ucx_devices any IMB-MPI1 pingpong