Improvements/#371 change nccl version (#372)

* Change nccl and efa installer in micro bench.

Close #371

* Fix mwemory size for nccl test on k8s.

Close #312

Author: mhuguesaws

micro-benchmarks/nccl-tests/README.md

@@ -36,17 +36,17 @@ The NCCL tests are packaged in a container.
 > | Variable              | Default     | Repository                                                                                  |
 > |-----------------------|-------------|---------------------------------------------------------------------------------------------|
 > |`GDRCOPY_VERSION`      | `v2.4.1`    | [link](https://github.com/NVIDIA/gdrcopy)                                                   |
-> |`EFA_INSTALLER_VERSION`| `1.31.0`    | [link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html#efa-start-enable) |
-> |`AWS_OFI_NCCL_VERSION` | `v1.8.1-aws`| [link](https://github.com/aws/aws-ofi-nccl)                                                 |
-> |`NCCL_VERSION`         | `v2.20.3-1` | [link](https://github.com/NVIDIA/nccl)                                                      |
+> |`EFA_INSTALLER_VERSION`| `1.33.0`    | [link](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html#efa-start-enable) |
+> |`AWS_OFI_NCCL_VERSION` | `v1.9.2-aws`| [link](https://github.com/aws/aws-ofi-nccl)                                                 |
+> |`NCCL_VERSION`         | `v2.21.5-1` | [link](https://github.com/NVIDIA/nccl)                                                      |
 > |`NCCL_TESTS_VERSION`   | `v2.13.9`   | [link](https://github.com/NVIDIA/nccl-tests)                                                |
 
 ### Build the container
 1. Build the container image with the command below:
    ```bash
-   EFA_INSTALLER_VERSION=1.31.0
-   AWS_OFI_NCCL_VERSION=v1.8.1-aws
-   NCCL_VERSION=v2.20.3-1
+   EFA_INSTALLER_VERSION=1.33.0
+   AWS_OFI_NCCL_VERSION=v1.9.2-aws
+   NCCL_VERSION=v2.21.5-1
    NCCL_TESTS_VERSION=v2.13.9
    docker build  -f nccl-tests.Dockerfile \
           --build-arg="EFA_INSTALLER_VERSION=${EFA_INSTALLER_VERSION}" \
@@ -81,9 +81,9 @@ To run the NCCL tests on EKS, you will need to build the container image, then p
 
 1. Create the ECR repository if it does not exist
    ```bash
-   EFA_INSTALLER_VERSION=1.31.0
-   AWS_OFI_NCCL_VERSION=v1.8.1-aws
-   NCCL_VERSION=v2.20.3-1
+   EFA_INSTALLER_VERSION=1.33.0
+   AWS_OFI_NCCL_VERSION=v1.9.2-aws
+   NCCL_VERSION=v2.21.5-1
    NCCL_TESTS_VERSION=v2.13.9
    ECR_REPOSITORY_NAME="nccl-tests"
    TAG="${EFA_INSTALLER_VERSION}-${AWS_OFI_NCCL_VERSION}-${NCCL_VERSION}-${NCCL_TESTS_VERSION}"

micro-benchmarks/nccl-tests/kubernetes/nccl-tests.yaml

@@ -88,12 +88,12 @@ spec:
                 nvidia.com/gpu: 8
                 hugepages-2Mi: 5120Mi
                 vpc.amazonaws.com/efa: 32
-                memory: 8000Mi
+                memory: 32000Mi
               requests:
                 nvidia.com/gpu: 8
                 hugepages-2Mi: 5120Mi
                 vpc.amazonaws.com/efa: 32
-                memory: 8000Mi
+                memory: 32000Mi
           volumes:
           - name: shmem
             hostPath:

micro-benchmarks/nccl-tests/nccl-tests.Dockerfile

@@ -3,9 +3,9 @@
 FROM nvidia/cuda:12.2.2-devel-ubuntu22.04
 
 ARG GDRCOPY_VERSION=v2.4.1
-ARG EFA_INSTALLER_VERSION=1.31.0
-ARG AWS_OFI_NCCL_VERSION=v1.8.1-aws
-ARG NCCL_VERSION=v2.20.3-1
+ARG EFA_INSTALLER_VERSION=1.33.0
+ARG AWS_OFI_NCCL_VERSION=v1.9.2-aws
+ARG NCCL_VERSION=v2.21.5-1
 ARG NCCL_TESTS_VERSION=v2.13.9
 
 RUN apt-get update -y && apt-get upgrade -y

micro-benchmarks/nccl-tests/slurm/nccl-tests-ami.sbatch

@@ -13,8 +13,8 @@ set -ex
 
 # This script is designed to run by default on the Deep Learning AMI, Ubuntu 20.04
 # See https://aws.amazon.com/releasenotes/aws-deep-learning-base-gpu-ami-ubuntu-20-04/
-ALL_REDUCE_BINARY=${1:-/usr/local/cuda-12.2/efa/test-cuda-12.2/all_reduce_perf}
-ADDITIONAL_LD_LIBRARY_PATH=${2:-}
+ALL_REDUCE_BINARY=${1:-/usr/local/cuda-12.3/efa/test-cuda-12.3/all_reduce_perf}
+ADDITIONAL_LD_LIBRARY_PATH=${2:-/usr/local/cuda-12.3/lib}
 
 # Get Hostname to Instance ID mapping
 mpirun -N 1 bash -c 'echo $(hostname) ➡️ $(cat /sys/devices/virtual/dmi/id/board_asset_tag | tr -d " ")'