Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 133 additions & 2 deletions configs/sites/tier1/navy-aws/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,134 @@
# Provisiong NRL ParallelWorks AWS clusters
# Preparing NRL ParallelWorks AWS clusters for spack-stack

TODO UPDATE 20251208 !!!
## Prerequisites

### GNU 13.4.0

Download and install:
```
module purge
umask 0022

mkdir -p /project/spack-stack/gcc-13.4.0/src
cd /project/spack-stack/gcc-13.4.0/src
wget https://github.com/gcc-mirror/gcc/archive/refs/tags/releases/gcc-13.4.0.tar.gz
tar -xf gcc-13.4.0.tar.gz

cd gcc-releases-gcc-13.4.0
./contrib/download_prerequisites 2>&1 | tee log.download_prerequisites
./configure \
--prefix=/project/spack-stack/gcc-13.4.0 \
--disable-multilib \
2>&1 | tee log.configure
make -j48 2>&1 | tee log.make
# Cannot run make check, because autogen is not installed
#make check 2>&1 | tee log.check
make install 2>&1 | tee log.install
```

Create modulefile `/project/spack-stack/gcc-13.4.0/modulefiles/gcc/13.4.0`:
```
mkdir -p /project/spack-stack/gcc-13.4.0/modulefiles/gcc
cat << EOF > /project/spack-stack/gcc-13.4.0/modulefiles/gcc/13.4.0
#%Module1.0

module-whatis "Provides gcc-13.4.0 for use with spack."

conflict gnu
conflict gcc

proc ModulesHelp { } {
puts stderr "Provides gcc-13.4.0 for use with spack."
}


# Set this value
set GCC_PATH "/project/spack-stack/gcc-13.4.0"

prepend-path PATH "${GCC_PATH}/bin"
prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib"
prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib64"
prepend-path LIBRARY_PATH "${GCC_PATH}/lib"
prepend-path LIBRARY_PATH "${GCC_PATH}/lib64"
prepend-path CPATH "${GCC_PATH}/include"
prepend-path CMAKE_PREFIX_PATH "${GCC_PATH}"
prepend-path PKG_CONFIG_PATH "${GCC_PATH}/usr/lib64/pkgconfig"
prepend-path MANPATH "${GCC_PATH}/share/man"
EOF
```

### OpenMPI 4.1.8

Download and install:
```
module purge
umask 0022

module use /project/spack-stack/gcc-13.4.0/modulefiles
module load gcc/13.4.0

mkdir -p /project/spack-stack/openmpi-4.1.8/gcc-13.4.0/src
cd /project/spack-stack/openmpi-4.1.8/gcc-13.4.0/src
wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.8.tar.gz
tar -xf openmpi-4.1.8.tar.gz

cd openmpi-4.1.8
./configure \
--prefix=/project/spack-stack/openmpi-4.1.8/gcc-13.4.0 \
2>&1 | tee log.configure
make -j48 2>&1 | tee log.make
make check 2>&1 | tee log.check
make install 2>&1 | tee log.install
```

Create modulefile `/project/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles/openmpi/4.1.8`:
```
mkdir -p /project/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles/openmpi
cd /project/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles/openmpi
cat <<EOF > /project/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles/openmpi/4.1.8
#%Module1.0

module-whatis "Provides openmpi-4.1.8 compiled with gcc-13.4.0 for use with spack."

conflict gnu
conflict gcc

proc ModulesHelp { } {
puts stderr "Provides openmpi-4.1.8 compiled with gcc-13.4.0 for use with spack."
}


# Set this value
set OPENMPI_PATH "/project/spack-stack/openmpi-4.1.8/gcc-13.4.0"

prepend-path PATH "${OPENMPI_PATH}/bin"
prepend-path LD_LIBRARY_PATH "${OPENMPI_PATH}/lib"
prepend-path LIBRARY_PATH "${OPENMPI_PATH}/lib"
prepend-path CPATH "${OPENMPI_PATH}/include"
prepend-path CMAKE_PREFIX_PATH "${OPENMPI_PATH}"
prepend-path MANPATH "${OPENMPI_PATH}/share/man"
EOF
```

### Intel oneAPI 2025.3.0

Download and install:
```
module purge
umask 022

mkdir -p /project/spack-stack/oneapi-2025.3.0/src
cd /project/spack-stack/oneapi-2025.3.0/src
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/66021d90-934d-41f4-bedf-b8c00bbe98bc/intel-oneapi-hpc-toolkit-2025.3.0.381_offline.sh

sh ./intel-oneapi-hpc-toolkit-2025.3.0.381_offline.sh -a --silent --cli --eula accept
```

Select a custom installation of all components in `/project/spack-stack/oneapi-2025.3.0`. After the installation, create the modulefiles:
```
cd /project/spack-stack/oneapi-2025.3.0
./modulefiles-setup.sh --ignore-latest --output-dir=/project/spack-stack/oneapi-2025.3.0/modulefiles

# Fix non-ascii characters in modulefiles
sed -i 's/®//g' `grep -lRe '®' modulefiles`
```
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ packages:
- "openmpi@4.1.8"
gcc:
externals:
- spec: gcc@13.3.1 languages:='c,c++,fortran'
prefix: /opt/rh/gcc-toolset-13/root
- spec: gcc@13.4.0 languages:='c,c++,fortran'
prefix: /project/spack-stack/gcc-13.4.0
modules:
- gcc/13.3.1
- gcc/13.4.0
extra_attributes:
compilers:
c: /opt/rh/gcc-toolset-13/root/bin/gcc
cxx: /opt/rh/gcc-toolset-13/root/bin/g++
fortran: /opt/rh/gcc-toolset-13/root/bin/gfortran
c: /project/spack-stack/gcc-13.4.0/bin/gcc
cxx: /project/spack-stack/gcc-13.4.0/bin/g++
fortran: /project/spack-stack/gcc-13.4.0/bin/gfortran
#- spec: gcc@11.5.0 languages:='c,c++,fortran'
# prefix: /usr
# extra_attributes:
Expand All @@ -23,12 +23,11 @@ packages:
# fortran: /usr/bin/gfortran
gcc-runtime:
externals:
- spec: gcc-runtime@13.3.1%gcc@13.3.1
prefix: /opt/rh/gcc-toolset-13/root
- spec: gcc-runtime@13.4.0%gcc@13.4.0
prefix: /project/spack-stack/gcc-13.4.0
openmpi:
externals:
- spec: openmpi@4.1.8
prefix: /project/spack-stack/openmpi-4.1.8/gcc-13.3.1
prefix: /project/spack-stack/openmpi-4.1.8/gcc-13.4.0
modules:
- openmpi/4.1.8

8 changes: 4 additions & 4 deletions util/nrl/batch_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ case ${SPACK_STACK_BATCH_HOST} in
SPACK_STACK_CARGO_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/cargo-mirror"
;;
navy-aws)
SPACK_STACK_BATCH_COMPILERS=("oneapi@=2025.3.0" "gcc@=13.3.1")
SPACK_STACK_BATCH_COMPILERS=("oneapi@=2025.3.0" "gcc@=13.4.0")
SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "cylc-dev")
SPACK_STACK_MODULE_CHOICE="tcl"
SPACK_STACK_BOOTSTRAP_MIRROR="/project/spack-stack/bootstrap-mirror"
Expand Down Expand Up @@ -450,9 +450,9 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do
umask 0022
module purge
case ${compiler} in
gcc-13.3.1)
module use /project/spack-stack/gcc-13.3.1/modulefiles
module use /project/spack-stack/openmpi-4.1.8/gcc-13.3.1/modulefiles
gcc-13.4.0)
module use /project/spack-stack/gcc-13.4.0/modulefiles
module use /project/spack-stack/openmpi-4.1.8/gcc-13.4.0/modulefiles
;;
esac
;;
Expand Down