From dc4d05fb70f6bb6c55444f042ce8f9996ac61932 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Dec 2025 19:02:38 -0700 Subject: [PATCH 01/12] Update for release/2.0: .gitmodules doc/CMakeLists.txt doc/source/conf.py --- .gitmodules | 4 ++-- doc/CMakeLists.txt | 2 +- doc/source/conf.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index b9fb10569..303cc47eb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = spack-stack-dev + branch = release/2.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules @@ -9,4 +9,4 @@ [submodule "repos/builtin"] path = repos/builtin url = https://github.com/jcsda/spack-packages - branch = spack-stack-dev + branch = release/2.0 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 0d21c6cf6..b738eb9c1 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.18 FATAL_ERROR) -project(spack-stack VERSION 1.9.0) +project(spack-stack VERSION 2.0.0) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/Modules") diff --git a/doc/source/conf.py b/doc/source/conf.py index bde663d23..a6c6aa351 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -24,10 +24,10 @@ author = 'spack-stack collaboration' # The short X.Y version -version = 'dev' +version = '2.0' # The full version, including alpha/beta/rc tags -release = 'develop' +release = '2.0.0' numfig = True From 0e5f24c9baa186fdee7867e79e6e769fb3b1325b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 9 Dec 2025 18:01:34 -0700 Subject: [PATCH 02/12] release/2.0 Bugfix for eckit/libzip, update nrl_batch_install.sh (-i fms), update common packages_clang.yaml (ip openmp), update navy-aws site config (#1838) For release/2.0 (will merge back to develop later): - Bug fix for eckit/libzip in spack-packages, see [release/2.0] Bug fix for eckit: handle new compression backend zip (libzip) for version 1.32+ spack-packages#26 - Add missing -i fms to show_duplicate_packages check in util/nrl/batch_install.sh - Configs/common/packages_clang.yaml: remove ~openmp variant for package ip - Update site config for tier-1 platform navy-aws (HSP ParallelWorks AWS) --- configs/common/packages_clang.yaml | 3 - configs/sites/tier1/navy-aws/README.md | 42 +----------- configs/sites/tier1/navy-aws/packages.yaml | 6 +- .../tier1/navy-aws/packages_gcc-13.3.1.yaml | 3 +- .../navy-aws/packages_oneapi-2024.2.1.yaml | 62 ------------------ .../navy-aws/packages_oneapi-2025.2.0.yaml | 64 ------------------- .../navy-aws/packages_oneapi-2025.3.0.yaml | 64 +++++++++++++++++++ repos/builtin | 2 +- util/nrl/batch_install.sh | 2 +- 9 files changed, 69 insertions(+), 179 deletions(-) delete mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml delete mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml create mode 100644 configs/sites/tier1/navy-aws/packages_oneapi-2025.3.0.yaml diff --git a/configs/common/packages_clang.yaml b/configs/common/packages_clang.yaml index 499e0963f..0d350dcc4 100644 --- a/configs/common/packages_clang.yaml +++ b/configs/common/packages_clang.yaml @@ -13,6 +13,3 @@ packages: ectrans: require: - +fftw - ip: - require: - - '~openmp' diff --git a/configs/sites/tier1/navy-aws/README.md b/configs/sites/tier1/navy-aws/README.md index 4adab23a4..6de88eea9 100644 --- a/configs/sites/tier1/navy-aws/README.md +++ b/configs/sites/tier1/navy-aws/README.md @@ -1,43 +1,3 @@ # Provisiong NRL ParallelWorks AWS clusters -TODO UPDATE 20250820 !!! - -## Steps to perform before installing spack-stack develop as of 2024/11/24 - -Note. Some of these packages may already be installed, but for the sake of completeness, they are listed here. -``` -sudo su - -chmod 777 /contrib - -yum install -y gcc-toolset-13 -yum install -y gcc-toolset-13-runtime -yum install -y gcc-toolset-13-binutils -yum install -y gcc-toolset-13-gcc -yum install -y gcc-toolset-13-gcc-c++ -yum install -y gcc-toolset-13-gcc-gfortran -yum install -y gcc-toolset-13-gdb - -yum install -y binutils-devel -yum install -y m4 -yum install -y wget -yum install -y git -yum install -y git-lfs -yum install -y bash-completion -yum install -y bzip2 bzip2-devel -yum install -y unzip -yum install -y patch -yum install -y automake -yum install -y xorg-x11-xauth -yum install -y xterm -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y texlive -yum install -y bison -yum install -y screen - -yum install -y qt5-qtbase -yum install -y qt5-qttools-devel -yum install -y qt5-qtsvg-devel -``` - -For instructions for building spack-stack, see the spack-stack documentation on readthedocs (https://spack-stack.readthedocs.io/en/latest). For instructions for using pre-built spack-stack environments, see the spack-stack wiki (https://github.com/JCSDA/spack-stack/wiki). +TODO UPDATE 20251208 !!! diff --git a/configs/sites/tier1/navy-aws/packages.yaml b/configs/sites/tier1/navy-aws/packages.yaml index e31c7b1ff..6357d6eec 100644 --- a/configs/sites/tier1/navy-aws/packages.yaml +++ b/configs/sites/tier1/navy-aws/packages.yaml @@ -106,11 +106,7 @@ packages: prefix: /usr libfabric: externals: - - spec: libfabric@2.1.0 fabrics=psm2,psm3,shm,tcp,verbs - prefix: /opt/intel/oneapi/mpi/2021.16 - - spec: libfabric@2.1.0 fabrics=psm2,psm3,shm,tcp,verbs - prefix: /opt/intel/oneapi/mpi/2021.16/opt/mpi/libfabric - - spec: libfabric@2.1.0 fabrics=psm2,psm3,shm,tcp,verbs + - spec: libfabric@1.22.0 fabrics=psm2,psm3,shm,tcp,verbs prefix: /usr libfuse: externals: diff --git a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml index 1a584cff1..b26bc7752 100644 --- a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml @@ -26,5 +26,4 @@ packages: openmpi: externals: - spec: openmpi@4.1.8 - prefix: /project/gcc/openmpi-4.1.8/gcc-13.3.1 - + prefix: /project/spack-stack/openmpi-4.1.8/gcc-13.3.1 diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml deleted file mode 100644 index 8ef29244e..000000000 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2024.2.1.yaml +++ /dev/null @@ -1,62 +0,0 @@ -packages: - mpi: - buildable: False - require: - - "intel-oneapi-mpi@2021.13" - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2024.2.1 - prefix: /project/oneapi/2024.2.1 - modules: - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 - extra_attributes: - compilers: - c: /project/oneapi/2024.2.1/compiler/2024.2/bin/icx - cxx: /project/oneapi/2024.2.1/compiler/2024.2/bin/icpx - fortran: /project/oneapi/2024.2.1/compiler/2024.2/bin/ifort - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.13 - prefix: /project/oneapi/2024.2.1 - modules: - - mpi/2021.13 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2024.2 - prefix: /project/oneapi/2024.2.1 - modules: - - mkl/2024.2 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /project/oneapi/2024.2.1 - modules: - - tbb/2021.13 - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2024.2.1 - prefix: /project/oneapi/2024.2.1 - modules: - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 - gcc: - externals: - - spec: gcc@11.5.0 languages:='c,c++' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.5.0 - prefix: /usr diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml deleted file mode 100644 index e76a98e50..000000000 --- a/configs/sites/tier1/navy-aws/packages_oneapi-2025.2.0.yaml +++ /dev/null @@ -1,64 +0,0 @@ -packages: - mpi: - buildable: False - require: - - "intel-oneapi-mpi@2021.16" - intel-oneapi-compilers: - externals: - - spec: intel-oneapi-compilers@2025.2.0 - prefix: /project/oneapi/2025.2.0 - modules: - - umf/0.11.0 - - tbb/2022.2 - - compiler-rt/2025.2.0 - - compiler/2025.2.0 - extra_attributes: - compilers: - c: /project/oneapi/2025.2.0/compiler/2025.2/bin/icx - cxx: /project/oneapi/2025.2.0/compiler/2025.2/bin/icpx - fortran: /project/oneapi/2025.2.0/compiler/2025.2/bin/ifx - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.16 - prefix: /project/oneapi/2025.2.0 - modules: - - mpi/2021.16 - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2025.2 - prefix: /project/oneapi/2025.2.0 - modules: - - mkl/2025.2 - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2022.2 - prefix: /project/oneapi/2025.2.0 - modules: - - tbb/2022.2 - intel-oneapi-runtime: - # LINK TO BUG REPORT HERE - #buildable: false - externals: - - spec: intel-oneapi-runtime@2025.2.0 - prefix: /project/oneapi/2025.2.0 - modules: - - umf/0.11.0 - - tbb/2022.2 - - compiler-rt/2025.2.0 - - compiler/2025.2.0 - gcc: - externals: - - spec: gcc@11.5.0 languages:='c,c++' - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - gcc-runtime: - externals: - - spec: gcc-runtime@11.5.0 - prefix: /usr diff --git a/configs/sites/tier1/navy-aws/packages_oneapi-2025.3.0.yaml b/configs/sites/tier1/navy-aws/packages_oneapi-2025.3.0.yaml new file mode 100644 index 000000000..26c5bd899 --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_oneapi-2025.3.0.yaml @@ -0,0 +1,64 @@ +packages: + mpi: + buildable: False + require: + - "intel-oneapi-mpi@2021.17" + intel-oneapi-compilers: + externals: + - spec: intel-oneapi-compilers@2025.3.0 + prefix: /project/spack-stack/oneapi-2025.3.0 + modules: + - umf/1.0 + - tbb/2022.3 + - compiler-rt/2025.3.0 + - compiler/2025.3.0 + extra_attributes: + compilers: + c: /project/spack-stack/oneapi-2025.3.0/compiler/2025.3/bin/icx + cxx: /project/spack-stack/oneapi-2025.3.0/compiler/2025.3/bin/icpx + fortran: /project/spack-stack/oneapi-2025.3.0/compiler/2025.3/bin/ifx + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.17 + prefix: /project/spack-stack/oneapi-2025.3.0 + modules: + - mpi/2021.17 + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.3 + prefix: /project/spack-stack/oneapi-2025.3.0 + modules: + - mkl/2025.3 + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.3 + prefix: /project/spack-stack/oneapi-2025.3.0 + modules: + - tbb/2022.3 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.3.0 + prefix: /project/spack-stack/oneapi-2025.3.0 + modules: + - umf/1.0 + - tbb/2022.3 + - compiler-rt/2025.3.0 + gcc: + buildable: false + externals: + - spec: gcc@11.5.0 languages:='c,c++' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + gcc-runtime: + buildable: false + externals: + - spec: gcc-runtime@11.5.0 + prefix: /usr diff --git a/repos/builtin b/repos/builtin index 9e174ef25..76b9f1974 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 9e174ef25efc1a8e3925cd6c9f5a75e81677e2c5 +Subproject commit 76b9f1974aee9776afb290bddd123fd31cac4fca diff --git a/util/nrl/batch_install.sh b/util/nrl/batch_install.sh index 4839b3a53..4cb3d5bb4 100755 --- a/util/nrl/batch_install.sh +++ b/util/nrl/batch_install.sh @@ -529,7 +529,7 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do fi # Check for duplicate packages - ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip + ./util/show_duplicate_packages.py -i crtm -i crtm-fix -i esmf -i mapl -i neptune-env -i py-cython -i ip -i fms # Update local source cache if requested if [[ "${update_source_cache}" == "true"* ]]; then From da94ac014c90bb578b2fc6b0a1549f6308256a89 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 11 Dec 2025 09:47:55 -0500 Subject: [PATCH 03/12] Update NAS README (#1841) --- configs/sites/tier1/nas/README.md | 273 ++++++++++++++++++++---------- 1 file changed, 184 insertions(+), 89 deletions(-) diff --git a/configs/sites/tier1/nas/README.md b/configs/sites/tier1/nas/README.md index 70aa6e8f8..28e3bbe6c 100644 --- a/configs/sites/tier1/nas/README.md +++ b/configs/sites/tier1/nas/README.md @@ -1,180 +1,275 @@ -# How to build spack-stack at NAS +# How to Build **spack-stack** at NAS -In the commands below some will be run on login nodes (with internet access) and some -on compute nodes as, at NAS, you aren't allowed more than 2 processes on a login node. +This guide documents how to build **spack-stack** on NASA NAS systems, where login nodes have internet access but are CPU-restricted, while compute nodes allow parallel builds but have *no* internet access. Several packages (Rust/Cargo, ecFlow, CRTM) require special handling due to these constraints. -## Machines +--- -For the below you will need to login to both an `afe01` node for one step. You'll -also want to get a Rome compute node for the rest of the steps. +## Table of Contents + +- [Overview](#overview) +- [Machines Required](#machines-required) +- [Clone spack-stack](#clone-spack-stack) +- [Obtain an Interactive Compute Node](#obtain-an-interactive-compute-node) +- [Setup spack-stack](#setup-spack-stack) +- [Create Environments](#create-environments) + - [oneAPI Environment](#oneapi-environment) + - [GCC Environment](#gcc-environment) +- [Activate the Environment](#activate-the-environment) +- [Concretize the Environment](#concretize-the-environment) +- [Create Source Cache (LOGIN NODE ONLY)](#create-source-cache-login-node-only) +- [Pre-Fetch Cargo Dependencies (LOGIN NODE ONLY)](#pre-fetch-cargo-dependencies-login-node-only) +- [Install Packages](#install-packages) + - [Step 1 — Dependencies of Rust codes and ecFlow (COMPUTE NODE)](#step-1--dependencies-of-rust-codes-and-ecflow-compute-node) + - [Step 2 — Rust codes and ecFlow (AFE LOGIN NODE)](#step-2--rust-codes-and-ecflow-afe-login-node) + - [Step 3 — Remaining Packages (COMPUTE NODE)](#step-3--remaining-packages-compute-node) + - [Packages Requiring Internet](#packages-requiring-internet) +- [Update Module Files](#update-module-files) +- [Deactivate the Environment](#deactivate-the-environment) +- [Debugging Package Builds](#debugging-package-builds) + +--- + +## Overview + +Due to NAS system architecture and network restrictions: + +- **Login nodes**: + - Have internet + - Limited to **2 processes** + - `pfe` nodes use **Sandy Bridge** (too old for x86_64_v3 builds) + +- **Compute nodes** (Milan / Rome): + - No internet + - Allow parallel builds + +Some packages (Cargo/Rust, ecFlow, CRTM) require internet or newer CPU features, so the install is broken into multiple steps across different node types. + +--- + +## Machines Required + +You will need: + +- **An `afe01` login node** + Supports x86_64_v3 binaries → required for building Rust packages and ecFlow. + +- **A Rome or Milan compute node** + Used for the main installation with multiple cores. + +--- ## Clone spack-stack -``` -git clone --recurse-submodules https://github.com/mathomp4/spack-stack.git -b feature/nas_install_spack_v1 spack-stack-2.0.0-test +Use the appropriate branch or tag: + +```bash +git clone --recurse-submodules https://github.com/JCSDA/spack-stack.git \ + -b spack-stack-2.0.0 spack-stack-2.0.0 ``` -## Grab interactive node +--- -Since NAS limits you to 2 processes on a login node, you'll need to grab an interactive node. For example: -``` -qsub -I -V -X -l select=1:ncpus=128:mpiprocs=128:model=mil_ait -l walltime=12:00:00 -W group_list=s1873 -m b -N Interactive +## Obtain an Interactive Compute Node + +NAS login nodes allow only **2 processes**, so use: + +```bash +qsub -I -V -X \ + -l select=1:ncpus=128:mpiprocs=128:model=mil_ait \ + -l walltime=12:00:00 \ + -W group_list=s1873 \ + -m b \ + -N Interactive ``` -will get you a Milan node for 12 hours -## Setup spack-stack on each node +This gives a **Milan** compute node for up to 12 hours. -We will start on a login node with internet access. This is mainly needed for the -`spack mirror create` command which downloads all the source code for the packages. +--- -``` -cd spack-stack-2.0.0-test +## Setup spack-stack + +Run on a **login node with internet**: + +```bash +cd spack-stack-2.0.0 . setup.sh ``` -## Create environments +--- -We create two different environments, one for oneAPI and one for GCC. The commands below -are used to create the environments. You only need to do this once. +## Create Environments -### oneAPI +You only need to create each environment once. -To create the oneAPI environment, do: +### oneAPI Environment -``` -spack stack create env --name ue-oneapi-2024.2.0 --template unified-dev --site nas --compiler=oneapi-2024.2.0 +```bash +spack stack create env --name ue-oneapi-2024.2.0 \ + --template unified-dev --site nas --compiler=oneapi-2024.2.0 cd envs/ue-oneapi-2024.2.0 ``` -### GCC - -To create the GCC environment, do: +### GCC Environment -``` -spack stack create env --name ue-gcc-13.2.0 --template unified-dev --site nas --compiler gcc-13.2.0 +```bash +spack stack create env --name ue-gcc-13.2.0 \ + --template unified-dev --site nas --compiler=gcc-13.2.0 cd envs/ue-gcc-13.2.0 ``` -## Activate environment +--- -Now enter the spack environment you just created: +## Activate the Environment -``` +```bash spack env activate . ``` -NOTE: You need to make sure you do this in *any* terminal where you want to do any commmand -below with this environment. +> **Important:** Run this in *every* terminal where you plan to run Spack commands. + +--- + +## Concretize the Environment -## Concretize and create source cache +Run on a **login node** (internet required for bootstrapping Clingo and other tools): +```bash +spack concretize 2>&1 | tee log.concretize ; bell ``` -spack concretize 2>&1 | tee log.concretize + +### Optional `bell` helper + +```bash +bell() { tput bel ; printf "\nFinished at: " ; date; } ``` -NOTE: The first time you do this on a new build, you should do it on a *LOGIN* node. This is because -it might need to bootstrap things and so it will reach out to the internet. +--- -## Create source cache (LOGIN NODE ONLY) +## Create Source Cache (LOGIN NODE ONLY) -Because this step downloads all the source code for all packages and all versions, it -should be done on a login node with internet access. +This downloads all source tarballs for your environment: -``` -spack mirror create -a -d /swbuild/gmao_SIteam/spack-stack/source-cache +```bash +spack mirror create -a \ + -d /swbuild/gmao_SIteam/spack-stack/source-cache ``` -NOTE: Make sure you are in an environment when you run that `spack mirror create` command. Otherwise, -you will download *EVERY* package and *EVERY* version in spack! +> ⚠️ **Do not run this outside an activated environment.** +> Otherwise Spack will attempt to mirror **every** known package/version. -## Pre-fetch cargo packages (LOGIN NODE ONLY) +--- -Some packages use Rust/Cargo for dependencies. These need internet access to build. So we pre-fetch them here. +## Pre-Fetch Cargo Dependencies (LOGIN NODE ONLY) -We need to set `CARGO_HOME` to a location where the Cargo deps have been downloaded +Rust packages frequently require network access during build. Pre-fetch their dependencies: -``` +```bash export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache ../../util/fetch_cargo_deps.py ``` -NOTE: `CARGO_HOME` should be set as well on the COMPUTE node! +> ⚠️ **You must also set `CARGO_HOME` on compute nodes** before building. -## Install packages +--- -Our install process will actually have (at least) three steps. This is because of the `crtm` package -which requires internet access at build time. +## Install Packages -### Install Step 1: Dependencies of Rust codes and ecflow (COMPUTE NODE) +Installation requires three stages: -We currently have some codes that use rust/cargo for dependencies. And, for some reason, -even doing the "cargo dependencies" as above, they still need internet -access to build/install. +| Step | Node Type | Why | +|------|-----------|-----| +| Step 1 | Compute | Build dependencies in parallel, avoids CPU limits | +| Step 2 | `afe` login | Needed for x86_64_v3 Python and internet access | +| Step 3 | Compute | Finish main installation at high parallelism | -As for ecflow, we built QT on a login node (as it was the only complete node), so we -then have to build ecflow on a login node as well. +--- -So we first install all the dependencies of then codes. +### Step 1 — Dependencies of Rust codes and ecFlow (COMPUTE NODE) -``` +```bash export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache -spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature --only dependencies py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.deps-for-rust-and-ecflow +spack install -j 16 --verbose --fail-fast --show-log-on-error \ + --no-check-signature \ + --only dependencies py-cryptography py-maturin py-rpds-py ecflow \ + 2>&1 | tee log.install.deps-for-rust-and-ecflow ; bell ``` -### Install Step 2: Rust Codes and ecflow (AFE LOGIN NODE) +--- -NOTE: You *MUST* run this on an afe login node. The reason is the pfe login nodes are Sandy -Bridge but we are building Spack with `x86_64_v3` and these are too old (`_v2`). So -you will get an illegal instruction error when the install below calls python3. +### Step 2 — Rust codes and ecFlow (AFE LOGIN NODE) -So go back to an afe login node and run: +`pfe` nodes use Sandy Bridge CPUs, which **cannot run** spack-stack’s x86_64_v3 Python interpreter → results in `Illegal instruction`. -``` +So this must be done on **afe**: + +```bash export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache -spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error --no-check-signature py-cryptography py-maturin py-rpds-py ecflow 2>&1 | tee log.install.rust-and-ecflow +spack install -j 2 -p 1 --verbose --fail-fast --show-log-on-error \ + --no-check-signature \ + py-cryptography py-maturin py-rpds-py ecflow \ + 2>&1 | tee log.install.rust-and-ecflow ; bell ``` -Note we are only using 2 processes here because NAS limits you to 2 processes on a login node. +NAS limits login nodes to 2 processes, hence `-j 2`. -### Install Step 3: The rest (COMPUTE NODE) +--- -``` +### Step 3 — Remaining Packages (COMPUTE NODE) + +```bash export CARGO_HOME=/swbuild/gmao_SIteam/spack-stack/cargo-cache -spack install -j 16 --verbose --fail-fast --show-log-on-error --no-check-signature 2>&1 | tee log.install.after-cargo +spack install -j 16 --verbose --fail-fast --show-log-on-error \ + --no-check-signature \ + 2>&1 | tee log.install.after-cargo ; bell ``` -NOTE: You might need to run the `spack install` command multiple times because sometimes -it just fails. But then you run it more and more and it will eventually succeed. +> **Note:** You may need to re-run this command multiple times. Some builds fail intermittently but succeed on retry. -### Packages needing internet access to build +--- -If you encounter other packages that need internet access to build, you can install them with: +### Packages Requiring Internet -``` -spack install -j 2 --verbose --fail-fast --show-log-on-error --no-check-signature |& tee log.install. +If you encounter another package that insists on network access: + +```bash +spack install -j 2 --verbose --fail-fast --show-log-on-error \ + --no-check-signature \ + |& tee log.install. ; bell ``` -Then, once that package is built, you can go back to the compute node and run the `spack install` command again. +Once built, return to the compute node and resume the full installation. -## Update module files and setup meta-modules +--- -``` -spack module tcl refresh -y --delete-tree +## Update Module Files + +After installation completes: + +```bash +spack module tcl refresh -y --delete-tree ; bell spack stack setup-meta-modules ``` -## Deactivate environment +--- -``` +## Deactivate the Environment + +```bash spack env deactivate ``` -# Debugging a package +--- -When things go wrong, a good way to debug a failure is: +## Debugging Package Builds -``` +```bash spack clean spack stage spack build-env -- bash --norc --noprofile ``` + +This drops you into a clean build environment with the package’s full compiler/runtime environment loaded. + +--- + + From e3778a8df53002803dad52b67e918f3b51f8a64a Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Thu, 11 Dec 2025 09:51:37 -0500 Subject: [PATCH 04/12] Set UDUNITS2_XML_PATH in modules (#1843) This is the "simple" solution to #1842. The ideal way will be to do so via spack-packages, but since spack-stack 2.0 is out soon, this is the "easy" way. We can undo this once/if spack/spack-packages#2729 is merged and then get into 2.1 --- configs/common/modules_lmod.yaml | 6 ++++++ configs/common/modules_tcl.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index a11068661..d4c09d01a 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -266,6 +266,12 @@ modules: set: 'UPP_INC': '{prefix}/include' 'UPP_LIB': '{prefix}/lib/libupp.a' + # NOTE: This will be fixed upstream in udunits Spack package + # Once this is done, remove this section + udunits: + environment: + set: + 'UDUNITS2_XML_PATH': '{prefix}/share/udunits/udunits2.xml' hierarchy: - mpi diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index b3fbb8ce8..2a02f2b47 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -285,3 +285,9 @@ modules: set: 'UPP_INC': '{prefix}/include' 'UPP_LIB': '{prefix}/lib/libupp.a' + # NOTE: This will be fixed upstream in udunits Spack package + # Once this is done, remove this section + udunits: + environment: + set: + 'UDUNITS2_XML_PATH': '{prefix}/share/udunits/udunits2.xml' From cdbdaea961bfad7de9f3ad577840d52ff55650fc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 12 Dec 2025 12:31:52 -0700 Subject: [PATCH 05/12] [release/2.0] Fix MET segmentation faults in spack-packages, navy-aws site config updates round 2 (#1844) - Fix MET segmentation faults (spack-packages submodule, see [release/2.0] Fix MET segfaults (fixup_rpath --> environment modification) spack-packages#29). This requires adding libpython3.* to the whitelist in util/ldd_check.py - Site config updates for navy-aws tier1 site, add site config to NRL batch install script --- configs/sites/tier1/navy-aws/bootstrap.yaml | 9 ------ configs/sites/tier1/navy-aws/config.yaml | 10 +++---- configs/sites/tier1/navy-aws/mirrors.yaml | 20 ++----------- configs/sites/tier1/navy-aws/packages.yaml | 8 ------ .../tier1/navy-aws/packages_gcc-13.3.1.yaml | 5 +++- repos/builtin | 2 +- util/ldd_check.py | 5 ++-- util/nrl/batch_install.sh | 28 +++++++++++++++++++ 8 files changed, 44 insertions(+), 43 deletions(-) delete mode 100644 configs/sites/tier1/navy-aws/bootstrap.yaml diff --git a/configs/sites/tier1/navy-aws/bootstrap.yaml b/configs/sites/tier1/navy-aws/bootstrap.yaml deleted file mode 100644 index 8602ff6ff..000000000 --- a/configs/sites/tier1/navy-aws/bootstrap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -bootstrap: - sources: - - name: local-binaries - metadata: /project/spack-stack/spack-bootstrap-mirror/metadata/binaries - - name: local-sources - metadata: /project/spack-stack/spack-bootstrap-mirror/metadata/sources - trusted: - local-sources: true - local-binaries: true diff --git a/configs/sites/tier1/navy-aws/config.yaml b/configs/sites/tier1/navy-aws/config.yaml index dd48c4196..50f07d3a4 100644 --- a/configs/sites/tier1/navy-aws/config.yaml +++ b/configs/sites/tier1/navy-aws/config.yaml @@ -1,10 +1,10 @@ config: - build_jobs: 6 + build_jobs: 24 # Overrides for spack build and staging areas to speed up builds # and avoid errors with hard links on the NFS filesystem /contrib - build_stage: ${HOME}/spack-stack-cache/build_stage - test_stage: ${HOME}/spack-stack-cache/test_stage - source_cache: ${HOME}/spack-stack-cache/source_cache - misc_cache: ${HOME}/spack-stack-cache/misc_cache + build_stage: /scratch/spack-stack-cache/build_stage + test_stage: /scratch/spack-stack-cache/test_stage + source_cache: /scratch/spack-stack-cache/source_cache + misc_cache: /scratch/spack-stack-cache/misc_cache diff --git a/configs/sites/tier1/navy-aws/mirrors.yaml b/configs/sites/tier1/navy-aws/mirrors.yaml index 3d247981f..696c473d2 100644 --- a/configs/sites/tier1/navy-aws/mirrors.yaml +++ b/configs/sites/tier1/navy-aws/mirrors.yaml @@ -1,18 +1,4 @@ mirrors: - local-source: - fetch: - url: file:///contrib/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///contrib/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///project/spack-stack/source-cache + local-binary: file:///project/spack-stack/build-cache + diff --git a/configs/sites/tier1/navy-aws/packages.yaml b/configs/sites/tier1/navy-aws/packages.yaml index 6357d6eec..03438b703 100644 --- a/configs/sites/tier1/navy-aws/packages.yaml +++ b/configs/sites/tier1/navy-aws/packages.yaml @@ -2,14 +2,6 @@ packages: # Modification of common packages # All other packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.2 - prefix: /usr bash: externals: - spec: bash@5.1.8 diff --git a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml index b26bc7752..8a7f3f4f8 100644 --- a/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml +++ b/configs/sites/tier1/navy-aws/packages_gcc-13.3.1.yaml @@ -26,4 +26,7 @@ packages: openmpi: externals: - spec: openmpi@4.1.8 - prefix: /project/spack-stack/openmpi-4.1.8/gcc-13.3.1 + #prefix: /project/spack-stack/openmpi-4.1.8/gcc-13.3.1 + modules: + - openmpi/4.1.8 + diff --git a/repos/builtin b/repos/builtin index 76b9f1974..f37a67b80 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit 76b9f1974aee9776afb290bddd123fd31cac4fca +Subproject commit f37a67b804eb966306ae3b8420cada9cf14d1c4a diff --git a/util/ldd_check.py b/util/ldd_check.py index 9cee8a838..a298474cb 100755 --- a/util/ldd_check.py +++ b/util/ldd_check.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 # Whitelist file patterns (checked with re.match()) that will be satisfied by -# compiler & MPI modules (though arguably these should be added as extra -# rpaths). +# compiler, MPI, and Python modules (though arguably these should be added as +# extra rpaths). whitelist = [ "^libmkl.+", "^libifcore.so.*", + "^libpython3.*", ] ######## diff --git a/util/nrl/batch_install.sh b/util/nrl/batch_install.sh index 4cb3d5bb4..d3ee94563 100755 --- a/util/nrl/batch_install.sh +++ b/util/nrl/batch_install.sh @@ -103,6 +103,11 @@ fi SPACK_STACK_BATCH_HOST=$(echo ${HOSTNAME} | cut -d "." -f 1) SPACK_STACK_BATCH_HOST=${SPACK_STACK_BATCH_HOST//[0-9]/} +# Workaround for ParallelWorks login nodes +if [[ "${SPACK_STACK_BATCH_HOST}" == *"awsneptunecluster"* ]]; then + SPACK_STACK_BATCH_HOST="navy-aws" +fi + case ${SPACK_STACK_BATCH_HOST} in atlantis) SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.1" "oneapi@=2025.3.0" "gcc@=13.4.0" "clang@=21.1.0") @@ -139,6 +144,13 @@ case ${SPACK_STACK_BATCH_HOST} in SPACK_STACK_BOOTSTRAP_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/bootstrap-mirror" SPACK_STACK_CARGO_MIRROR="/p/cwfs/projects/NEPTUNE/spack-stack/cargo-mirror" ;; + navy-aws) + SPACK_STACK_BATCH_COMPILERS=("oneapi@=2025.3.0" "gcc@=13.3.1") + SPACK_STACK_BATCH_TEMPLATES=("neptune-dev" "cylc-dev") + SPACK_STACK_MODULE_CHOICE="tcl" + SPACK_STACK_BOOTSTRAP_MIRROR="/project/spack-stack/bootstrap-mirror" + SPACK_STACK_CARGO_MIRROR="/project/spack-stack/cargo-mirror" + ;; tusk) SPACK_STACK_BATCH_COMPILERS=("oneapi@=2024.2.0" "gcc@=12.1.0") SPACK_STACK_BATCH_TEMPLATES=("neptune-dev") @@ -222,6 +234,13 @@ function fix_permissions() { fi nice -n 19 lfs find ${dir} -type f -print0 | xargs --null chmod a+r ;; + navy-aws) + nice -n 19 find ${dir} -type d -print0 | xargs --null chmod a+rx + if [[ ${executables} -eq 1 ]]; then + nice -n 19 find ${dir} -type f -executable -print0 | xargs --null chmod a+rx + fi + nice -n 19 find ${dir} -type f -print0 | xargs --null chmod a+r + ;; tusk) nice -n 19 lfs find ${dir} -type d -print0 | xargs --null chmod a+rx # In case the find command returns no executables @@ -427,6 +446,15 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do umask 0022 module purge ;; + navy-aws) + umask 0022 + module purge + case ${compiler} in + gcc-13.3.1) + module use /project/spack-stack/openmpi-4.1.8/gcc-13.3.1/modulefiles + ;; + esac + ;; tusk) umask 0022 set +e From 57d03d805f13d0e8c04d527656dd1eafda7e94c3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 17 Dec 2025 07:58:11 -0700 Subject: [PATCH 06/12] release/2.0: Update batch_install.sh - workaround for ParallelWorks (#1849) Update batch_install.sh - workaround for ParallelWorks: no NRL GitHub access (yet), turn off adp variant. --- util/nrl/batch_install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/nrl/batch_install.sh b/util/nrl/batch_install.sh index d3ee94563..215b505e2 100755 --- a/util/nrl/batch_install.sh +++ b/util/nrl/batch_install.sh @@ -491,6 +491,9 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do fi spack env activate -p ${env_dir} + # Workaround for ParallelWorks (no NRL Enterprise GitHub access yet) + sed -i 's/+adp/~adp/g' ${env_dir}/spack.yaml + # Update bootstrap mirror if requested if [[ "${update_bootstrap_mirror}" == "true"* ]]; then tmp_bootstrap_mirror_path=${PWD}/tmp-bootstrap-mirror-${env_name} From 625993374525ecb0d801f6716bf3cd6cfd370d3f Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Thu, 18 Dec 2025 09:13:46 -0700 Subject: [PATCH 07/12] EC2 ubuntu site config for 2.0 release. (#1851) In addition to including the site config for the 2.0 release this also includes the py-scipy config change discussed in #1848 Finally, the rocky8 config is fairly out of date, I'm removing it for now. --- configs/common/packages_oneapi.yaml | 3 + configs/sites/tier2/aws-rocky8/README.md | 457 ------------------ configs/sites/tier2/aws-rocky8/compilers.yaml | 28 -- configs/sites/tier2/aws-rocky8/config.yaml | 9 - configs/sites/tier2/aws-rocky8/modules.yaml | 7 - configs/sites/tier2/aws-rocky8/packages.yaml | 94 ---- .../sites/tier2/aws-rocky8/packages_gcc.yaml | 5 - .../tier2/aws-rocky8/packages_oneapi.yaml | 27 -- configs/sites/tier2/aws-ubuntu2404/README.md | 232 +++++---- .../sites/tier2/aws-ubuntu2404/compilers.yaml | 29 -- .../sites/tier2/aws-ubuntu2404/packages.yaml | 84 ++-- .../tier2/aws-ubuntu2404/packages_gcc.yaml | 14 +- .../tier2/aws-ubuntu2404/packages_oneapi.yaml | 78 ++- 13 files changed, 233 insertions(+), 834 deletions(-) delete mode 100644 configs/sites/tier2/aws-rocky8/README.md delete mode 100644 configs/sites/tier2/aws-rocky8/compilers.yaml delete mode 100644 configs/sites/tier2/aws-rocky8/config.yaml delete mode 100644 configs/sites/tier2/aws-rocky8/modules.yaml delete mode 100644 configs/sites/tier2/aws-rocky8/packages.yaml delete mode 100644 configs/sites/tier2/aws-rocky8/packages_gcc.yaml delete mode 100644 configs/sites/tier2/aws-rocky8/packages_oneapi.yaml delete mode 100644 configs/sites/tier2/aws-ubuntu2404/compilers.yaml diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml index efc461a49..f620a636e 100644 --- a/configs/common/packages_oneapi.yaml +++ b/configs/common/packages_oneapi.yaml @@ -43,6 +43,9 @@ packages: py-maturin: require: - '%gcc' + py-scipy: + require: + - 'cxxflags="-O1"' qt: require: - '%gcc' diff --git a/configs/sites/tier2/aws-rocky8/README.md b/configs/sites/tier2/aws-rocky8/README.md deleted file mode 100644 index f2c227d0d..000000000 --- a/configs/sites/tier2/aws-rocky8/README.md +++ /dev/null @@ -1,457 +0,0 @@ -## spack-stack AMI (Rocky8) - -This document is to go over the the running and usage of this specific AMI for Rocky8. This image has two environments: gnu (gcc-11.2), and intel (intel@2021.10.0). You can use either one of these environments for development purposes. - -### Using the AMI - -Here is an example `aws cli` command-line to run this AMI with your designated key-pair. - -```bash -aws ec2 run-instances \ - --image-id "ami-02391db2758465a87" --instance-type "m6i.4xlarge" \ - --key-name YOUR-KEYPAIR \ - --block-device-mappings '{"DeviceName":"/dev/sda1","Ebs":{"Encrypted":false,"DeleteOnTermination":true,"Iops":3000,"SnapshotId":"snap-05fb00e35af5550e7","VolumeSize":150,"VolumeType":"gp3","Throughput":125}}' \ - --network-interfaces '{"SubnetId":"subnet-00575f6b8ccc2005d","AssociatePublicIpAddress":true,"DeviceIndex":0,"Groups":["sg-0436b207bc220df08"]}' \ - --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"rocky8-spack-stack-1.9-gcc-oneapi"}]}' \ - --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ - --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \ - --count "1" -``` - -## Using this Site Config - -JCSDA publishes a fully configured and built installation of spack stack derived -from this config as an Amazon Machine Images (AMI). The easiest way to use this -configuration of spack stack is to launch a VM using that AMI. The instructions -below are included for maintainance of the history of this site config and -their possible relevance to debugging issues should they arise. - -### Base Instance - -This AMI was built on an instance with these properties: - -* AMI Name: Rocky-8-EC2-Base-8.9-20231119.0.x86_64-d6577ceb-8ea8-4e0e-84c6-f098fc302e82 -* AMI ID: ami-02391db2758465a87 -* Instance m6i.4xlarge (uses Intel Xeon processor) -* 300GB of gp3 storage as / - -## Pre-requisites (For All Compilers) - -
-Tmux Usage - -### Tmux - -Rocky 8 doesn't have screen so here's a guide to tmux. - -```bash -# Start a session -tmux new -s my-session-name - -# Detach from a session -ctr-b d - -# list sessions -tmux ls - -# Attach to existing session -tmux attach -t my-session-name -``` - -More information on how to use `tmux`: - -
- -### Installing Packages - -```bash -# Update system software and start a tmux session. -sudo dnf -y update -sudo dnf -y install tmux -tmux new -s setup -sudo su - - -# Compilers -dnf install -y gcc-toolset-11-gcc-c++ \ - gcc-toolset-11-gcc-gfortran \ - gcc-toolset-11-gdb - -#Install other requirements. -dnf install -y binutils-devel \ - m4 \ - wget \ - git \ - git-lfs \ - bash-completion \ - bzip2 bzip2-devel \ - libcurl libcurl-devel \ - unzip \ - patch \ - automake \ - xorg-x11-xauth \ - xterm \ - perl \ - gettext-devel \ - texlive \ - tcl-devel \ - nano \ - vim \ - bison \ - qt5-qtbase \ - qt5-qtbase-devel \ - qt5-qtsvg \ - qt5-qtsvg-devel \ - libxml2 libxml2-devel - -# Python develop. -dnf install -y python3-devel - -# Enable gcc toolset. This is needed for later builds. Once lmod is installed -# this will be configured as a pass-through module. -scl enable gcc-toolset-11 bash - -# Configure git credential caching and git lfs for the rocky user and root. -git config --global credential.helper 'cache --timeout=3600' -git lfs install - -# Configure x11 forwarding. -echo "X11Forwarding yes" >> /etc/ssh/sshd_config -service sshd restart -``` - -### Install MySQL Community Server - -```bash -dnf config-manager --set-enabled crb - -dnf install -y mysql-server mysql-devel -sudo systemctl start mysqld.service -sudo systemctl enable mysqld - -# Use the mysql server. -sudo mysql -u root -USE mysql; -ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; -FLUSH PRIVILEGES; -EXIT; -exit -``` - -### Install Lmod - -```bash -# Enable gcc using system module -scl enable gcc-toolset-11 bash - -# Install lua/lmod manually because apt only has older versions -# that are not compatible with the modern lua modules spack produces -# https://lmod.readthedocs.io/en/latest/030_installing.html#install-lua-x-y-z-tar-gz -sudo su - -mkdir -p /opt/lua/5.1.4.9/src && cd $_ -wget https://sourceforge.net/projects/lmod/files/lua-5.1.4.9.tar.bz2 -tar -xvf lua-5.1.4.9.tar.bz2 -cd lua-5.1.4.9 -./configure --prefix=/opt/lua/5.1.4.9 2>&1 | tee log.config -make VERBOSE=1 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cat << 'EOF' >> /etc/profile.d/02-lua.sh -# Set environment variables for lua -export PATH="/opt/lua/5.1.4.9/bin:$PATH" -export LD_LIBRARY_PATH="/opt/lua/5.1.4.9/lib:$LD_LIBRARY_PATH" -export CPATH="/opt/lua/5.1.4.9/include:$CPATH" -export MANPATH="/opt/lua/5.1.4.9/man:$MANPATH" -EOF - -source /etc/profile.d/02-lua.sh -mkdir -p /opt/lmod/8.7/src -cd /opt/lmod/8.7/src -wget https://sourceforge.net/projects/lmod/files/Lmod-8.7.tar.bz2 -tar -xvf Lmod-8.7.tar.bz2 -cd Lmod-8.7 -# Note the weird prefix, lmod installs in PREFIX/lmod/X.Y automatically -./configure --prefix=/opt/ \ - --with-lmodConfigDir=/opt/lmod/8.7/config \ - 2>&1 | tee log.config -make install 2>&1 | tee log.install -ln -sf /opt/lmod/lmod/init/profile /etc/profile.d/z00_lmod.sh -ln -sf /opt/lmod/lmod/init/cshrc /etc/profile.d/z00_lmod.csh -ln -sf /opt/lmod/lmod/init/profile.fish /etc/profile.d/z00_lmod.fish - -# Add custom module for system gcc -cat << 'EOF' >> /opt/rh/gcc-toolset-11/gcc-toolset.lua ---%Module -family("compiler") -help([[Wrapper module for gcc-toolset-11 using Software Collections]]) -whatis("Description: GCC Toolset 11") - --- Execute 'scl enable' to load gcc-toolset-11 environment -execute {cmd="scl enable gcc-toolset-11 bash", modeA={"load"}} -EOF - -# Add a number of default module locations to the lmod startup script. -cat << 'EOF' >> /etc/profile.d/z01_lmod.sh -module use /opt/rh/gcc-toolset-11 -EOF - -# Log out completely, ssh back into the instance and check if lua modules work -exit -exit -``` - -### Clone `spack-stack` - -```bash -# if in sudo su - -cd /opt -git clone -b develop --recursive https://github.com/jcsda/spack-stack.git -``` - -## Install Steps by Compiler - -
-GCC Installation - -```bash -sudo su - - -cd /opt/spack-stack -source setup.sh - -scl enable gcc-toolset-11 bash - -# Swap default module type for default linux. -sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml -spack stack create env --site linux.default --template unified-dev --name unified-env-gcc --compiler gcc -cd envs/unified-env-gcc -spack env activate -p . -export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" -spack external find --scope system \ - --exclude cmake \ - --exclude curl --exclude openssl \ - --exclude openssh --exclude python -spack external find --scope system wget -spack external find --scope system mysql -spack external find --scope system grep -spack compiler find --scope system -unset SPACK_SYSTEM_CONFIG_PATH - -# ACTION: Edit the site/packages.yaml and add these packages -# If not present. -cat << 'EOF' >> $PWD/site/packages.yaml - gcc: - buildable: false - externals: - - spec: gcc@11.2.1 - prefix: /usr - gcc-runtime: - buildable: false - externals: - - spec: gcc-runtime@11.2.1 - prefix: /usr - qt: - buildable: false - externals: - - spec: qt@5.15.3 - prefix: /usr - version: [5.15.3] -EOF - -# Continue configuration. -spack config add "packages:all:compiler:[gcc@11.2.1]" -spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" -spack config add "packages:fontconfig:variants:+pic" -spack config add "packages:pixman:variants:+pic" -spack config add "packages:cairo:variants:+pic" -spack config add "packages:ewok-env:variants:+mysql" - -# Concretize and install -spack concretize 2>&1 | tee log.concretize -${SPACK_STACK_DIR}/util/show_duplicate_packages.py -spack install --fail-fast -j 12 2>&1 | tee log.install - -# Install modules -spack module lmod refresh -spack stack setup-meta-modules - -cat << 'EOF' >> /etc/profile.d/z01_lmod.sh -module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core -EOF -``` - -
- -
-Intel OneAPI Installation - -#### Install Intel OneAPI Compiler - -```bash -rm -rf /opt/intel -rm -rf /var/intel - -tee > /tmp/oneAPI.repo << EOF -[oneAPI] -name=Intel® oneAPI repository -baseurl=https://yum.repos.intel.com/oneapi -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB -EOF - -sudo mv /tmp/oneAPI.repo /etc/yum.repos.d - -dnf install -y intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -``` - -#### Create Intel OneAPI Environment -```bash -tmux -s intel -sudo su - -scl enable gcc-toolset bash - -source /opt/intel/oneapi/setvars.sh - -cd /opt/spack-stack -source ./setup.sh - -spack stack create env --site linux.default --template unified-dev --name unified-env-oneapi --compiler oneapi -cd envs/unified-env-oneapi -spack env activate -p . - -export SPACK_SYSTEM_CONFIG_PATH="${PWD}/site" - -spack external find --scope system --exclude python --exclude curl -spack external find --scope system perl -spack external find --scope system wget -spack external find --scope system texlive -spack external find --scope system mysql -spack external find --scope system grep - -# No external find for pre-installed intel-oneapi-mpi (from pcluster AMI), -# and no way to add object entry to list using "spack config add". -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - gcc: - buildable: false - externals: - - spec: gcc@11.2.1 - prefix: /usr - gcc-runtime: - buildable: false - externals: - - spec: gcc-runtime@11.2.1 - prefix: /usr - intel-oneapi-mkl: - buildable: false - externals: - - spec: intel-oneapi-mkl@2024.2 - prefix: /opt/intel/oneapi - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 - prefix: /opt/intel/oneapi - intel-oneapi-tbb: - buildable: false - externals: - - spec: intel-oneapi-tbb@2021.13 - prefix: /opt/intel/oneapi - intel-oneapi-runtime: - buildable: false - externals: - - spec: intel-oneapi-runtime%oneapi@2024.2.1 - prefix: /opt/intel/oneapi -EOF - -# Can't find qt5 because qtpluginfo is broken, -# and no way to add object entry to list using "spack config add". -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - qt: - buildable: false - externals: - - spec: qt@5.15.3 - prefix: /usr -EOF - -spack compiler find --scope system - -unset SPACK_SYSTEM_CONFIG_PATH - -# spack config add "packages:mpi:buildable:False" -spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" -spack config add "packages:all:compiler:[oneapi@2024.2.1, gcc@11.2.1]" - - -spack concretize 2>&1 | tee log.concretize -${SPACK_STACK_DIR}/util/show_duplicate_packages.py -spack install -j 12 --verbose 2>&1 | tee log.install -spack module lmod refresh -spack stack setup-meta-modules - -cat << 'EOF' >> /etc/profile.d/z01_lmod.sh -module use /opt/spack-stack/envs/unified-env-gcc/install/modulefiles/Core -EOF -``` - -
- -## Test Installation - -
-GCC - -```bash -# Example given for building jedi-bundle -module use /opt/spack-stack/envs/unified-dev-gcc/install/modulefiles/Core -module load stack-gcc/11.2.1 -module load stack-openmpi/5.0.5 -module load base-env -module load jedi-mpas-env -module load jedi-fv3-env -module load ewok-env -module load sp - -mkdir /opt/jedi -cd /opt/jedi -git clone https://github.com/JCSDA-internal/jedi-bundle.git -cd jedi-bundle -mkdir build && cd build -ecbuild ../ -make update -make -j10 -ctest -``` - -
- -
-Intel OneAPI - -```bash -# Example given for building jedi-bundle -module use /opt/spack-stack/envs/unified-dev-oneapi/install/modulefiles/Core -module load stack-oneapi/2024.2.1 -module load stack-intel-oneapi-mpi/2021.13 -module load base-env -module load jedi-mpas-env -module load jedi-fv3-env -module load ewok-env -module load sp - -mkdir /opt/jedi -cd /opt/jedi -git clone https://github.com/JCSDA-internal/jedi-bundle.git -cd jedi-bundle -mkdir build && cd build -ecbuild ../ -make update -make -j10 -ctest -``` - -**Note**: If the `make -j10` shows any errors, re-run it and it should be successful. - -
- -The installation and configuration is now complete for the instance. diff --git a/configs/sites/tier2/aws-rocky8/compilers.yaml b/configs/sites/tier2/aws-rocky8/compilers.yaml deleted file mode 100644 index 81a2c28b6..000000000 --- a/configs/sites/tier2/aws-rocky8/compilers.yaml +++ /dev/null @@ -1,28 +0,0 @@ -compilers: -- compiler: - spec: oneapi@=2024.2.1 - paths: - cc: /opt/intel/oneapi/compiler/2024.2/bin/icx - cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort - fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@11.2.1 - paths: - cc: /opt/rh/gcc-toolset-11/root/user/bin/gcc - cxx: /opt/rh/gcc-toolset-11/root/user/bin/g++ - f77: /opt/rh/gcc-toolset-11/root/user/bin/gfortran - fc: /opt/rh/gcc-toolset-11/root/user/bin/gfortran - flags: {} - operating_system: rocky8 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - diff --git a/configs/sites/tier2/aws-rocky8/config.yaml b/configs/sites/tier2/aws-rocky8/config.yaml deleted file mode 100644 index cf54e6360..000000000 --- a/configs/sites/tier2/aws-rocky8/config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -config: - build_jobs: 8 - - # Overrides for spack build and staging areas to speed up builds - # by using a local directory instead of the EFS shared filesystem - build_stage: /tmp/spack-stack/cache/build_stage - test_stage: /tmp/spack-stack/cache/test_stage - source_cache: /tmp/spack-stack/cache/source_cache - misc_cache: /tmp/spack-stack/cache/misc_cache diff --git a/configs/sites/tier2/aws-rocky8/modules.yaml b/configs/sites/tier2/aws-rocky8/modules.yaml deleted file mode 100644 index 65801706f..000000000 --- a/configs/sites/tier2/aws-rocky8/modules.yaml +++ /dev/null @@ -1,7 +0,0 @@ -modules: - default: - enable:: - - lmod - lmod: - include: - - python diff --git a/configs/sites/tier2/aws-rocky8/packages.yaml b/configs/sites/tier2/aws-rocky8/packages.yaml deleted file mode 100644 index 6158d0119..000000000 --- a/configs/sites/tier2/aws-rocky8/packages.yaml +++ /dev/null @@ -1,94 +0,0 @@ -packages: - mysql: - externals: - - spec: mysql@8.0.37 - prefix: /usr - wget: - externals: - - spec: wget@1.21.2 - prefix: /usr - binutils: - externals: - - spec: binutils@2.38 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.32 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.8 - prefix: /usr - findutils: - externals: - - spec: findutils@4.8.0 - prefix: /usr - gawk: - externals: - - spec: gawk@5.1.0 - prefix: /usr - tar: - externals: - - spec: tar@1.34 - prefix: /usr - autoconf: - externals: - - spec: autoconf@2.71 - prefix: /usr - automake: - externals: - - spec: automake@1.16.5 - prefix: /usr - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /usr - gettext: - externals: - - spec: gettext@0.21 - prefix: /usr - git: - externals: - - spec: git@2.34.1~tcltk - prefix: /usr - git-lfs: - externals: - - spec: git-lfs@3.0.2 - prefix: /usr - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - groff: - externals: - - spec: groff@1.22.4 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - perl: - externals: - - spec: perl@5.34.0~cpanm+opcode+open+shared+threads - prefix: /usr - pkg-config: - externals: - - spec: pkg-config@0.29.2 - prefix: /usr - qt: - buildable: false - externals: - - spec: qt@5.15.3 - prefix: /usr - sed: - externals: - - spec: sed@4.8 - prefix: /usr - lcms: - externals: - - spec: lcms@2.13.1 - prefix: /usr diff --git a/configs/sites/tier2/aws-rocky8/packages_gcc.yaml b/configs/sites/tier2/aws-rocky8/packages_gcc.yaml deleted file mode 100644 index 8ee1dc22b..000000000 --- a/configs/sites/tier2/aws-rocky8/packages_gcc.yaml +++ /dev/null @@ -1,5 +0,0 @@ -packages: - all: - compiler:: [gcc@11.2.1] - providers: - mpi:: [openmpi@4.1.6] diff --git a/configs/sites/tier2/aws-rocky8/packages_oneapi.yaml b/configs/sites/tier2/aws-rocky8/packages_oneapi.yaml deleted file mode 100644 index f557efe46..000000000 --- a/configs/sites/tier2/aws-rocky8/packages_oneapi.yaml +++ /dev/null @@ -1,27 +0,0 @@ -packages: - all: - compiler:: [oneapi@2024.2.1,gcc@11.2.1] - providers: - mpi:: [intel-oneapi-mpi@2023.13] - # Remove the next three lines to switch to intel-oneapi-mkl - blas:: [openblas] - fftw-api:: [fftw] - lapack:: [openblas] - intel-oneapi-mpi: - buildable: false - externals: - - spec: intel-oneapi-mpi@2021.13 - prefix: /opt/intel/oneapi - intel-oneapi-mkl: - buildable: False - ectrans: - require:: - - '@1.5.0 ~mkl +fftw' - gsibec: - require:: - - '@1.2.1 ~mkl' - py-numpy: - require:: - - '@:1.26.4 ^openblas' - gmake: - buildable: False diff --git a/configs/sites/tier2/aws-ubuntu2404/README.md b/configs/sites/tier2/aws-ubuntu2404/README.md index 86efa4698..46c7a046a 100644 --- a/configs/sites/tier2/aws-ubuntu2404/README.md +++ b/configs/sites/tier2/aws-ubuntu2404/README.md @@ -15,7 +15,7 @@ aws ec2 run-instances \ --tag-specifications '{"ResourceType":"instance","Tags":[{"Key":"Name","Value":"ubuntu2404-spack-stack-1.9-gcc-oneapi"}]}' \ --metadata-options '{"HttpEndpoint":"enabled","HttpPutResponseHopLimit":2,"HttpTokens":"required"}' \ --private-dns-name-options '{"HostnameType":"ip-name","EnableResourceNameDnsARecord":false,"EnableResourceNameDnsAAAARecord":false}' \ - --count "1" + --count 1 ``` ## Using this Site Config @@ -46,16 +46,16 @@ apt update apt upgrade -y # Build tools -apt install -y build-essential g++-11 gcc-11 gfortran-11 g++-12 gcc-12 gfortran-12 g++-13 gcc-13 gfortran-13 make cmake automake autoconf apt-utils +apt install -y build-essential g++-13 gcc-13 gfortran-13 make cmake automake autoconf apt-utils #Install other requirements. -apt install -y cpp-11 libgomp1 git git-lfs autopoint mysql-server libmysqlclient-dev qtbase5-dev qt5-qmake libqt5svg5-dev qt5dxcb-plugin wget curl file tcl-dev gnupg2 iproute2 locales unzip less bzip2 gettext libtree pkg-config libcurl4-openssl-dev mysql-server libtool flex llvm-14 +apt install -y cpp-13 libgomp1 git git-lfs autopoint mysql-server libmysqlclient-dev qtbase5-dev qt5-qmake libqt5svg5-dev qt5dxcb-plugin wget curl file tcl-dev gnupg2 iproute2 locales unzip less bzip2 gettext libtree pkg-config libcurl4-openssl-dev mysql-server libtool flex llvm-14 # Set llvm config. update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 10 # Editors -apt install -y vim nano +apt install -y vim nano # Python develop. apt install -y python3 python3-pip python3-setuptools @@ -63,11 +63,10 @@ apt install -y python3 python3-pip python3-setuptools # Configure git credential caching and git lfs for the rocky user and root. git config --global credential.helper 'cache --timeout=3600' git lfs install - -# Change the gcc, g++, and gfortran version to 11 and give it the highest priority -update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 -update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 -update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-11 100 +# Change the gcc, g++, and gfortran version to 13 and give it the highest priority +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 +update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 exit # Exit root access ``` @@ -117,27 +116,11 @@ exit exit ``` -### Setup MySQL Sercer - -```bash -sudo systemctl status mysql.service -sudo systemctl start mysql.service # Only if it isn't running - -# Use the mysql server. -# This sets mysql up for root passwordless access (required by R2D2) -# Make sure this machine and database are not accessible via public networks -sudo mysql -u root -USE mysql; -ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; -FLUSH PRIVILEGES; -EXIT; -``` - ### Clone `spack-stack` ```bash cd /opt -sudo git clone -b develop --depth 1 --recursive https://github.com/jcsda/spack-stack.git +sudo git clone -b release/2.0 --depth 1 --recursive https://github.com/jcsda/spack-stack.git ``` ## Install Spack-Stack Steps by Compiler @@ -152,18 +135,20 @@ cd /opt/spack-stack source setup.sh # Swap default module type for default linux. sed -i 's/tcl/lmod/g' configs/sites/tier2/linux.default/modules.yaml -spack stack create env --site linux.default --template unified-dev --name unified-env-gcc --compiler gcc -cd envs/unified-env-gcc +sed -i "s/- '%gcc'/- '%gcc_toolchain'/" ./common/packages.yaml +spack stack create env --site linux.default --template unified-dev --name unified-gcc --compiler gcc +cd envs/unified-gcc spack env activate -p . -export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" +unset SPACK_DISABLE_LOCAL_CONFIG +export SPACK_SYSTEM_CONFIG_PATH="$(pwd)/site" spack external find --scope system \ --exclude cmake \ --exclude curl --exclude openssl \ --exclude openssh --exclude python spack external find --scope system wget -spack external find --scope system mysql spack external find --scope system grep spack compiler find --scope system +export SPACK_DISABLE_LOCAL_CONFIG=true unset SPACK_SYSTEM_CONFIG_PATH # ACTION: Edit the site/compilers.yaml with the following. # 1) Delete or comment gcc-13 refs and preserve only gcc-12 @@ -190,17 +175,17 @@ cat << 'EOF' >> $PWD/site/packages.yaml EOF # Continue configuration. -spack config add "packages:all:compiler:[gcc@11.4.0]" -spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" +spack config add "packages:all:prefer:['%gcc']" +spack config add "packages:all:providers:mpi:[openmpi@5.0.8]" spack config add "packages:fontconfig:variants:+pic" spack config add "packages:pixman:variants:+pic" spack config add "packages:cairo:variants:+pic" -spack config add "packages:ewok-env:variants:+mysql" +spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" # Concretize and install spack concretize 2>&1 | tee log.concretize -${SPACK_STACK_DIR}/util/show_duplicate_packages.py -spack install --fail-fast -j 12 2>&1 | tee log.install +${SPACK_STACK_DIR}/util/show_duplicate_packages.py -i fms -i crtm -i crtm-fix -i esmf -i mapl -i py-cython +spack install --fail-fast -j 14 2>&1 | tee log.install # Install modules spack module lmod refresh @@ -224,8 +209,8 @@ sudo su - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list -apt update -apt install intel-oneapi-compiler-dpcpp-cpp-2024.2 intel-oneapi-compiler-fortran-2024.2 intel-oneapi-mpi-devel-2021.13 intel-oneapi-tbb-devel-2021.13 intel-oneapi-mkl-devel-2024.2 -y +apt update +apt install intel-oneapi-compiler-dpcpp-cpp-2025.3 intel-oneapi-compiler-fortran-2025.3 intel-oneapi-mpi-devel-2021.17 intel-oneapi-tbb-devel-2022.3 intel-oneapi-mkl-devel-2025.3 -y exit ``` @@ -236,6 +221,7 @@ exit sudo su - # Create all modulefiles. /opt/intel/oneapi/modulefiles-setup.sh --output-dir=/opt/intel/oneapi/modulefiles +module use /opt/intel/oneapi/modulefiles # Add the oneapi module files to lmod init (confirm that this file does not exist) cat << 'EOF' >> /etc/profile.d/z01_lmod.sh @@ -244,23 +230,22 @@ EOF # Create combined module file. mkdir /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/ -cat << 'EOF' >> /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/2024.2.1 +cat << 'EOF' >> /opt/intel/oneapi/modulefiles/intel-oneapi-full-env/2025.3.0 #%Module1.0 ## -## intel-oneapi-full-env/2024.2.1 +## intel-oneapi-full-env/2025.3.0 ## Intel oneAPI full module environment proc ModulesHelp { } { puts stderr "intel-oneapi-full-env defines the entire module set used for spack-stack intel builds" } module-whatis "intel-oneapi-full-env defines the entire module set used for spack-stack intel builds" -module load tbb/2021.13 -module load compiler-rt/2024.2.1 -module load compiler/2024.2.1 -module load ifort/2024.2.1 -module load mpi/2021.13 -module load mkl/2024.2 -module load compiler-intel-llvm/2024.2.1 +module load umf/1.0.2 +module load tbb/2022.3 +module load compiler-rt/2025.3.0 +module load compiler/2025.3.0 +module load mkl/2025.3 +module load compiler-intel-llvm/2025.3.0 EOF ``` @@ -269,83 +254,149 @@ EOF ```bash sudo su - -module load intel-oneapi-full-env/2024.2.1 +module load intel-oneapi-full-env/2025.3.0 +export FC=ifx +export CXX=icpx +export CC=icx cd /opt/spack-stack source ./setup.sh -spack stack create env --site linux.default --template unified-dev --name unified-env-oneapi --compiler oneapi -cd envs/unified-env-oneapi +spack stack create env --site linux.default --template unified-dev --name unified-oneapi --compiler oneapi +cd envs/unified-oneapi spack env activate -p . -export SPACK_SYSTEM_CONFIG_PATH="${PWD}/site" +# Before finding packages you need to go into ./common/packages.yaml +# and comment out gmake requirements to prevent gmake from being +# over-constrained. +# pico ./common/packages.yaml +# Near the top of the file find and comment out these three lines. +# gmake: +# require: +# - '%gcc' + + +# Find external packages for the site config. +unset SPACK_DISABLE_LOCAL_CONFIG +export SPACK_SYSTEM_CONFIG_PATH="$(pwd)/site" spack external find --scope system --exclude bison --exclude openssl --exclude python --exclude gettext --exclude m4 --exclude cmake --exclude curl -spack external find --scope system perl spack external find --scope system wget -spack external find --scope system texlive -spack external find --scope system mysql spack external find --scope system grep -# No external find for pre-installed intel-oneapi-mpi, -# and no way to add object entry to list using "spack config add". +# Here we are doing some manual configuration to address the +# following tricky situations +# - External find doesn't work well for pre-installed intel-oneapi-mpi +# and we are using an external module load for this. +# - Specify correct gcc runtime. +# - Disable "buildable" on all intel modules. cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - gcc: - buildable: false - externals: - - spec: gcc@11.4.0 - prefix: /usr gcc-runtime: buildable: false externals: - - spec: gcc-runtime@11.4.0 - prefix: /usr + - spec: gcc-runtime@13.3.0 + prefix: /usr intel-oneapi-mkl: buildable: false externals: - - spec: intel-oneapi-mkl@2024.2 + - spec: intel-oneapi-mkl@2025.3 prefix: /opt/intel/oneapi + modules: + - mkl/2025.3 intel-oneapi-mpi: buildable: false externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 + - spec: intel-oneapi-mpi@2021.17 prefix: /opt/intel/oneapi + modules: + - mpi/2021.17 intel-oneapi-tbb: buildable: false externals: - - spec: intel-oneapi-tbb@2021.13 + - spec: intel-oneapi-tbb@2022.3 prefix: /opt/intel/oneapi + modules: + - tbb/2022.3 intel-oneapi-runtime: buildable: false externals: - - spec: intel-oneapi-runtime%oneapi@2024.2.1 + - spec: intel-oneapi-runtime@2025.3.0 prefix: /opt/intel/oneapi -EOF - -# Can't find qt5 because qtpluginfo is broken, -# and no way to add object entry to list using "spack config add". -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - qt: - buildable: false - externals: - - spec: qt@5.15.3 - prefix: /usr + modules: + - compiler-rt/2025.3.0 EOF spack compiler find --scope system -# Edit site/compilers.yaml -pico ${PWD}/site/compilers.yaml -# 1) Replace ifx with ifort. -# 2) Add "- intel-oneapi-full-env/2024.2.1" to the modules section for oneapi. -# 3) Disable gcc compilers other than 11.4.0 -sed -i 's/ifx/ifort/g' ${PWD}/site/compilers.yaml - +# Edit site/packages.yaml +pico ${PWD}/site/packages.yaml +# Your intel compiler should look something like this below +# and any non-preferred GCC toolchains should be removed. Note +# that GCC should reference languages c, c++, but not fortran. +# +# NOTE! watch out for redundant empty gcc specs which will cause +# unintelligible build errors later. There should only be one spec +# under "externals:". +# +# intel-oneapi-compilers: +# buildable: false +# externals: +# - spec: intel-oneapi-compilers@2025.3.0 +# prefix: /opt/intel/oneapi +# modules: +# - umf/1.0.2 +# - tbb/2022.3 +# - compiler-rt/2025.3.0 +# - compiler/2025.3.0 +# extra_attributes: +# compilers: +# c: /opt/intel/oneapi/compiler/2025.3/bin/icx +# fortran: /opt/intel/oneapi/compiler/2025.3/bin/ifx +# cxx: /opt/intel/oneapi/compiler/2025.3/bin/icpx +# gcc: +# buildable: false +# externals: +# - spec: gcc@13.3.0 languages:='c,c++,fortran' +# .... + +export SPACK_DISABLE_LOCAL_CONFIG=true unset SPACK_SYSTEM_CONFIG_PATH -spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" -spack config add "packages:all:compiler:[oneapi@2024.2.1, gcc@11.4.0]" -spack config add "packages:gmake:buildable:False" +# Edit the spack.yaml to include these clauses. +cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + all: + prefer: + - '%oneapi' + conflict: + - '%c=oneapi %fortran=gcc' + - '%c,cxx=oneapi %fortran=gcc' + - '%c=gcc %fortran=oneapi' + - '%c,cxx=gcc %fortran=oneapi' + - '%fortran=oneapi %c=gcc' + - '%fortran=oneapi %c,cxx=gcc' + - '%fortran=gcc %c,cxx=oneapi' + - '%fortran=gcc %c=oneapi' + providers: + mpi: [intel-oneapi-mpi@2021.17] + met: + variants: +python +grib2 +graphics +lidar2nc +modis + mpi: + buildable: false + require: + - intel-oneapi-mpi@2021.17 + py-scipy: + require: + #- '%c,cxx,fortran=gcc' + - 'cxxflags="-O1"' + jedi-base-env: + require: + - ~bufrquery + - +fftw + - +hdf4 + py-pyyaml: + require: + - +libyaml +EOF spack concretize 2>&1 | tee log.concretize ${SPACK_STACK_DIR}/util/show_duplicate_packages.py @@ -393,13 +444,10 @@ ctest Intel OneAPI ```bash -# Re-source the intel OneAPI environment -module load intel-oneapi-full-env/2024.2.1 - -# Example given for building jedi-bundle +# Build jedi-bundle with oneapi module use /opt/spack-stack/envs/unified-env-oneapi/install/modulefiles/Core -module load stack-oneapi/2024.2.1 -module load stack-intel-oneapi-mpi/2021.13 +module load stack-intel-oneapi-compilers/2025.3.0 +module load stack-intel-oneapi-mpi/2021.17 module load base-env module load jedi-mpas-env module load jedi-fv3-env diff --git a/configs/sites/tier2/aws-ubuntu2404/compilers.yaml b/configs/sites/tier2/aws-ubuntu2404/compilers.yaml deleted file mode 100644 index 78358a1ae..000000000 --- a/configs/sites/tier2/aws-ubuntu2404/compilers.yaml +++ /dev/null @@ -1,29 +0,0 @@ -compilers: -- compiler: - spec: oneapi@=2024.2.1 - paths: - cc: /opt/intel/oneapi/compiler/2024.2/bin/icx - cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort - fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort - flags: {} - operating_system: ubuntu24.04 - target: x86_64 - modules: - - intel-oneapi-full-env/2024.2.1 - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@11.4.0 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: {} - operating_system: ubuntu24.04 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - diff --git a/configs/sites/tier2/aws-ubuntu2404/packages.yaml b/configs/sites/tier2/aws-ubuntu2404/packages.yaml index 9e9415d47..1b22e06af 100644 --- a/configs/sites/tier2/aws-ubuntu2404/packages.yaml +++ b/configs/sites/tier2/aws-ubuntu2404/packages.yaml @@ -1,35 +1,41 @@ packages: - mysql: + gmake: + buildable: false externals: - - spec: mysql@8.0.41 + - spec: gmake@4.3 prefix: /usr - wget: + gcc-runtime: + buildable: false externals: - - spec: wget@1.21.4 + - spec: gcc-runtime@13.3.0 prefix: /usr - binutils: + grep: externals: - - spec: binutils@2.38 + - spec: grep@3.11 prefix: /usr - coreutils: + wget: externals: - - spec: coreutils@8.32 + - spec: wget@1.21.4 prefix: /usr - diffutils: + perl: externals: - - spec: diffutils@3.10 + - spec: perl@5.38.2~cpanm+opcode+open+shared+threads prefix: /usr findutils: externals: - spec: findutils@4.9.0 prefix: /usr - gawk: + coreutils: externals: - - spec: gawk@5.1.0 + - spec: coreutils@9.4 prefix: /usr - tar: + pkgconf: externals: - - spec: tar@1.34 + - spec: pkgconf@1.8.1 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.7 prefix: /usr autoconf: externals: @@ -39,61 +45,27 @@ packages: externals: - spec: automake@1.16.5 prefix: /usr - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.21 + - spec: gawk@5.2.1 prefix: /usr git: externals: - spec: git@2.43.0~tcltk prefix: /usr - git-lfs: - externals: - - spec: git-lfs@3.4.1 - prefix: /usr - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - groff: - externals: - - spec: groff@1.22.4 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.7 - prefix: /usr - m4: - externals: - - spec: m4@1.4.19 - prefix: /usr - perl: - externals: - - spec: perl@5.38.2~cpanm+opcode+open+shared+threads - prefix: /usr - pkg-config: - externals: - - spec: pkg-config@0.29.2 - prefix: /usr - qt: - buildable: false + flex: externals: - - spec: qt@5.15.3 + - spec: flex@2.6.4+lex prefix: /usr sed: externals: - spec: sed@4.9 prefix: /usr - lcms: + diffutils: externals: - - spec: lcms@2.13.1 + - spec: diffutils@3.10 prefix: /usr - llvm: - buildable: false + groff: externals: - - spec: llvm@14.0.6 + - spec: groff@1.23.0 prefix: /usr diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml index 1ffc73cbb..70fff3a47 100644 --- a/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml +++ b/configs/sites/tier2/aws-ubuntu2404/packages_gcc.yaml @@ -1,5 +1,11 @@ packages: - all: - compiler:: [gcc@11.4.0] - providers: - mpi:: [openmpi@5.0.6] + gcc: + externals: + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + buildable: false diff --git a/configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml b/configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml index 29c27707a..63eec2fc2 100644 --- a/configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml +++ b/configs/sites/tier2/aws-ubuntu2404/packages_oneapi.yaml @@ -1,32 +1,58 @@ packages: - all: - compiler:: [oneapi@2024.2.1, gcc@11.4.0] - providers: - mpi:: [intel-oneapi-mpi@2021.13] - # Remove the next three lines to switch to intel-oneapi-mkl - blas:: [openblas] - fftw-api:: [fftw] - lapack:: [openblas] - intel-oneapi-runtime: - buildable: False + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.3.0 + prefix: /opt/intel/oneapi + modules: + - umf/1.0.2 + - tbb/2022.3 + - compiler-rt/2025.3.0 + - compiler/2025.3.0 + extra_attributes: + compilers: + fortran: /opt/intel/oneapi/compiler/2025.3/bin/ifx + c: /opt/intel/oneapi/compiler/2025.3/bin/icx + cxx: /opt/intel/oneapi/compiler/2025.3/bin/icpx + openmpi: + buildable: false + mpich: + buildable: false + gcc: externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + buildable: false + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.3 prefix: /opt/intel/oneapi + modules: + - mkl/2025.3 intel-oneapi-mpi: - buildable: False + buildable: false externals: - - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 + - spec: intel-oneapi-mpi@2021.17 + modules: + - mpi/2021.17 prefix: /opt/intel/oneapi - intel-oneapi-mkl: - buildable: False - ectrans: - require:: - - '@1.5.0 ~mkl +fftw' - gsibec: - require:: - - '@1.2.1 ~mkl' - py-numpy: - require:: - - '@:1.26.4 ^openblas' - gmake: - buildable: False \ No newline at end of file + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.3 + prefix: /opt/intel/oneapi + modules: + - tbb/2022.3 + intel-oneapi-runtime: + buildable: false + externals: + - spec: intel-oneapi-runtime@2025.3.0 + prefix: /opt/intel/oneapi + modules: + - compiler-rt/2025.3.0 From 9e599826b412dc28036863962988eec570239a7d Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 18 Dec 2025 15:20:16 -0500 Subject: [PATCH 08/12] Acorn/WCOSS2/NCO updates (#1850) * put NCO-approved pkg list under wcoss2 site config instead of nco template * acorn deployments.yaml: use @ * allow setup.sh to run from outside of spack-stack dir structure * update repos/builtin/ submodule * debug harfbuzz (disable cmake) --------- Co-authored-by: Dom Heinzeller --- configs/common/packages.yaml | 4 + configs/sites/tier1/acorn/deployments.yaml | 4 +- configs/sites/tier1/acorn/modules.yaml | 2 + configs/sites/tier1/acorn/packages.yaml | 3 +- .../tier1/wcoss2/approved_packages.txt} | 26 +- configs/sites/tier1/wcoss2/modules.yaml | 2 + configs/sites/tier1/wcoss2/packages.yaml | 3 +- repos/builtin | 2 +- setup.sh | 2 +- util/acorn/deploy/deploy.py | 314 ------------------ 10 files changed, 30 insertions(+), 332 deletions(-) rename configs/{templates/nco/approved_list.txt => sites/tier1/wcoss2/approved_packages.txt} (92%) delete mode 100755 util/acorn/deploy/deploy.py diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index d80145ea6..83c8e3561 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -161,6 +161,10 @@ packages: #gsl-lite: # require: # - '@0.37.0' + harfbuzz: + # cmake-based harfbuzz fails during build + require: + - one_of: ['build_system=meson', 'build_system=autotools'] hdf: require: - '@4.2.15' diff --git a/configs/sites/tier1/acorn/deployments.yaml b/configs/sites/tier1/acorn/deployments.yaml index f1d54a1ac..30912cfb4 100644 --- a/configs/sites/tier1/acorn/deployments.yaml +++ b/configs/sites/tier1/acorn/deployments.yaml @@ -1,6 +1,6 @@ deployments: - template: "unified-dev" - compilers: ["oneapi-2024.2.1", "intel-19.1.3.304"] + compilers: ["oneapi@2024.2.1", "intel@19.1.3.304"] packages_to_install: - ufs-weather-model-env - global-workflow-env @@ -20,7 +20,7 @@ deployments: - py-numpy - template: "nco" - compilers: ["oneapi-2024.2.1", "intel-19.1.3.304"] + compilers: ["oneapi@2024.2.1", "intel@19.1.3.304"] duplicates_to_ignore: ["crtm", "crtm-fix", "boost", "harfbuzz"] allowed_gcc_packages: - antlr diff --git a/configs/sites/tier1/acorn/modules.yaml b/configs/sites/tier1/acorn/modules.yaml index 65801706f..93dffdd39 100644 --- a/configs/sites/tier1/acorn/modules.yaml +++ b/configs/sites/tier1/acorn/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: include: - python + all: + autoload: run diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index 65042c7fe..db9201d45 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -12,7 +12,7 @@ packages: blas: require:: ['openblas'] fftw-api: - require: ['fftw'] + require:: ['fftw'] jpeg: require: ['libjpeg-turbo'] lapack: @@ -39,6 +39,7 @@ packages: require: ['~crypto'] nco: require:: ['%oneapi'] + py-colorama:: {} py-pandas: require:: ['~excel'] subversion: diff --git a/configs/templates/nco/approved_list.txt b/configs/sites/tier1/wcoss2/approved_packages.txt similarity index 92% rename from configs/templates/nco/approved_list.txt rename to configs/sites/tier1/wcoss2/approved_packages.txt index 3fcdd8216..7eaf169a8 100644 --- a/configs/templates/nco/approved_list.txt +++ b/configs/sites/tier1/wcoss2/approved_packages.txt @@ -2,9 +2,9 @@ awscli bacio boost bufr -bufr_dump -bufr_tranjb -bufr_util +bufr-dump +bufr-tranjb +bufr-util cairo cdo cfitsio @@ -28,14 +28,14 @@ gempak geos gfsio git -grib_util +grib-util gsl hdf5 imagemagick ip ip2 jasper -lamp_shared +lamp-shared landsfcutil libdap4 libgeotiff @@ -54,19 +54,20 @@ mbedtls met metis metplus -mos_shared +mos-shared nco ncview nemsio nemsiogfs -netcdf +netcdf-c +netcdf-fortran perl pigz pio pixman prepobs -prod_envir -prod_util +prod-envir +prod-util proj python readline @@ -79,11 +80,11 @@ sp subversion udunits upp -util_shared +util-shared w3emc w3nco wgrib2 -wrf_io +wrf-io xyz2shp xz zlib @@ -210,7 +211,7 @@ xproto xtrans xxhash zstd -atlas +ecmwf-atlas eckit fckit gsl-lite @@ -223,3 +224,4 @@ glibc parallelio pkg-config ecbuild +gsi-ncdiag diff --git a/configs/sites/tier1/wcoss2/modules.yaml b/configs/sites/tier1/wcoss2/modules.yaml index 68cdd1e90..18f7d2444 100644 --- a/configs/sites/tier1/wcoss2/modules.yaml +++ b/configs/sites/tier1/wcoss2/modules.yaml @@ -11,6 +11,8 @@ modules: - netcdf_c_virtual - esmf_virtual - python_virtual + all: + autoload: run boost: suffixes: '%gcc': gcc diff --git a/configs/sites/tier1/wcoss2/packages.yaml b/configs/sites/tier1/wcoss2/packages.yaml index 6a16bae7c..a266b09dd 100644 --- a/configs/sites/tier1/wcoss2/packages.yaml +++ b/configs/sites/tier1/wcoss2/packages.yaml @@ -12,7 +12,7 @@ packages: blas: require:: ['openblas'] fftw-api: - require: ['fftw'] + require:: ['fftw'] jpeg: require: ['libjpeg-turbo'] lapack: @@ -55,6 +55,7 @@ packages: require:: ['%oneapi'] netcdf-c: require:: ['@4.9.2', '+mpi', '~parallel-netcdf', '~dap', '~blosc', '~szip'] + py-colorama:: {} py-pandas: require:: ['~excel'] scotch: diff --git a/repos/builtin b/repos/builtin index f37a67b80..58a2d586f 160000 --- a/repos/builtin +++ b/repos/builtin @@ -1 +1 @@ -Subproject commit f37a67b804eb966306ae3b8420cada9cf14d1c4a +Subproject commit 58a2d586f9aa965d2257cc3a43633e94fec61405 diff --git a/setup.sh b/setup.sh index 1aa3c09c4..e88fda426 100755 --- a/setup.sh +++ b/setup.sh @@ -25,7 +25,7 @@ echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env. echo "Changing bootstrap path to $(spack bootstrap root '$spack/bootstrap')" # Get the current hash of the spack-stack code -export SPACK_STACK_HASH=`git rev-parse --short HEAD` +export SPACK_STACK_HASH=`cd $SPACK_STACK_DIR && git rev-parse --short HEAD` echo "Current hash of spack-stack is ${SPACK_STACK_HASH}" ## Register the spack extension diff --git a/util/acorn/deploy/deploy.py b/util/acorn/deploy/deploy.py deleted file mode 100755 index 1d349a147..000000000 --- a/util/acorn/deploy/deploy.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env spack-python - -import argparse - -parser = argparse.ArgumentParser(description="Your script description") - -parser.add_argument('-n', '--no-scheduler', action='store_true', help="Run installation on local node (no job scheduler)") -parser.add_argument('-x', '--skip-go-rust-handling', action='store_true', help="Skip handling of Go/Rust dep fetching when using parallel job scheduler") -parser.add_argument('-r', '--redeploy-existing', action='store_true', help="Redeploy existing deployments (default is skip existing env dirs)") -parser.add_argument('-s', '--site', type=str, help='Site name override') -parser.add_argument('-u', '--until', choices=("create", "concretize", "validate", "fetch", "install"), help='Carry out steps up to and including') - -parser.add_argument('deployments', nargs='*', help="List of deployments to apply (default is all; specify template+compiler with, e.g., 'unified-dev%%oneapi@2024.2.1')") - -args = parser.parse_args() - -import collections -from datetime import datetime -import logging -import os -import socket -import subprocess -import sys -import yaml -from contextlib import redirect_stdout, redirect_stderr -from types import SimpleNamespace - -logging.basicConfig(level=logging.CRITICAL) - -import spack.extensions -spack.extensions.load_extension("stack") -from spack.extensions.stack.cmd.stack_cmds.create import * -from spack.extensions.stack.meta_modules import setup_meta_modules -import spack.environment -from spack.cmd.concretize import concretize -from spack.cmd.fetch import fetch -from spack.cmd.module import module -from spack import modules -from spack.util.executable import which - -spack_stack_dir = os.getenv("SPACK_STACK_DIR") - -sys.path.append(os.path.join(spack_stack_dir, "util")) -from show_duplicate_packages import show_duplicate_packages - -nowdate = datetime.now().strftime("%Y%m%d-%H%M") -logdir = os.path.join(spack_stack_dir, "deploy_logs") -os.makedirs(logdir, exist_ok=True) - -def get_site_and_tier(deployment={}): - if args.site: - return args.site, "tier1" - if "site" in deployment: - return deployment["site"], "tier1" - fqdn = socket.getfqdn() - if "acorn.wcoss2" in fqdn: - return "acorn", "tier1" - -def get_env_dir_basename(deployment): - base = deployment["template"] - base = base.replace("unified-dev", "ue") - base = base.replace("-dev", "") - return "-".join([base, deployment["compiler"]]) - -def deployment_already_exists(env_dir_basename, args): - path_to_check = os.path.join(spack_stack_dir, "envs", env_dir_basename) - return os.path.isdir(path_to_check) - -def is_deployment_requested(env_dir_basename, deployment, args): - template = deployment["template"] - template_and_compiler = deployment["template"] + "%" + deployment["compiler"] - if args.deployments and (template not in args.deployments) and (template_and_compiler not in args.deployments): - return False - if template in args.deployments: - return True - if template_and_compiler in args.deployments: - return True - if args.redeploy_existing: - return True - return not deployment_already_exists(env_dir_basename, args) - -def get_create_env_settings(env_dir_basename, deployment, deployments): - config_dict = {} - config_dict["site"] = get_site_and_tier(deployment=deployment)[0] - config_dict["template"] = deployment["template"] - config_dict["dir"] = os.path.join(spack_stack_dir, "envs") - config_dict["name"] = env_dir_basename - if "upstreams" in deployment: - upstream_full_paths = [] - for upstream_template in deployment["upstreams"]: - for _candidate_upstream in deployments.values(): - if (_candidate_upstream["template"] == upstream_template) and (_candidate_upstream["compiler"] == deployment["compiler"]): - upstream_basename = get_env_dir_basename(_candidate_upstream) - upstream_full_path = os.path.join(spack_stack_dir, "envs", upstream_basename, "install") - upstream_full_paths.append([upstream_full_path]) - break - config_dict["upstreams"] = upstream_full_paths - config_dict["compiler"] = deployment["compiler"] - - return config_dict - -def run_batch_install(batch_config, deployment, env_dir_full_path, logfile, logfilepath, packages_to_install=[], suffix=".batch_install"): - if "walltime" in deployment: - walltime = deployment["walltime"] - elif "default_walltime" in batch_config: - walltime = batch_config["default_walltime"] - else: - assert False, "Set deployment-specific walltime or batch_config:default_walltime" - - if batch_config["scheduler"] == "pbspro": - cmd = [ - "qsub", - "-o", logfilepath + suffix, - "-e", logfilepath + suffix, - "-A", batch_config["account"], - "-q", batch_config["queue"], - "-l", "walltime=" + walltime + ",select=1:ncpus=12", - "-V", "-Wblock=true", "--", - which("spack").path, "--env", env_dir_full_path, - "install", "--fail-fast", "--show-log-on-error", - "--concurrent-packages", "4", "--jobs", "4", - ] - else: - assert False, "batch_config:scheduler must be pbspro" - if packages_to_install: - cmd.extend(packages_to_install) - logfile.write("Launching batch job:\n%s\n" % " ".join(cmd)) - subprocess.run(cmd, stdout=logfile, stderr=logfile, check=True) - -assert not os.getenv("SPACK_ENV"), "$SPACK_ENV is set. Reconsider your choices." - -# Load deployments.yaml configuration -site, tier = get_site_and_tier() -deployments_yaml_path = os.path.join(spack_stack_dir, "configs", "sites", tier, site, "deployments.yaml") -with open(deployments_yaml_path, "r") as f: - deployments_yaml = yaml.safe_load(f) -print(f"Loading deployments.yaml for site {site}") - -# Generate deployments object, including iterating over compilers -deployments = collections.OrderedDict() - -for _deployment in deployments_yaml["deployments"]: - for _compiler in _deployment["compilers"]: - deployment = _deployment.copy() - del(deployment["compilers"]) - deployment["compiler"] = _compiler - if "packages_to_install" not in deployment: - deployment["packages_to_install"] = [] - if "only_concretize_requested_packages" not in deployment: - deployment["only_concretize_requested_packages"] = False - env_dir_basename = get_env_dir_basename(deployment) - deployments[env_dir_basename] = deployment - print(f" Registered deployment: {deployment['template']}/{deployment['compiler']} ({env_dir_basename})") - -print("="*30) - -# Create and install each deployment -for env_dir_basename, deployment in deployments.items(): - if not is_deployment_requested(env_dir_basename, deployment, args): - print(f"Skipping deployment: {deployment['template']}/{deployment['compiler']} ({env_dir_basename})") - continue - print("="*30) - # Create env based on config - stack_settings = get_create_env_settings(env_dir_basename, deployment, deployments) - env_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename) - if os.path.isdir(env_dir_full_path) and args.redeploy_existing: - backup_dir_full_path = os.path.join(spack_stack_dir, "envs", env_dir_basename + "_bkp") - assert not os.path.isdir(backup_dir_full_path), f"Backup dir {backup_dir_full_path} already exists" - print(f"Moving {env_dir_full_path}\n to {backup_dir_full_path}") - os.rename(env_dir_full_path, backup_dir_full_path) - logfilepath = os.path.join(logdir, nowdate + f".{deployment['template']}.{deployment['compiler']}.log") - print(f"Log file: {logfilepath}") - logfile = open(logfilepath, "a", buffering=1) - logfile.write(str(deployment) + "\n") - logfile.write(str(stack_settings) + "\n") - print(f"Creating environment for {deployment['template']}/{deployment['compiler']}") - print(f" at {env_dir_full_path} ...") - # Use equivalent of 'spack stack create env' - stack_env = StackEnv(**stack_settings) - stack_env.write() - stack_env.check_umask() - env = spack.environment.Environment(env_dir_full_path) - spack.environment.activate(env) - - # Filter out unwanted packages before concretization - if deployment["only_concretize_requested_packages"]: - for root_spec in env.roots(): - if root_spec.name not in deployment["packages_to_install"]: - env.remove(root_spec) - env.write() - - if args.until == "create": - spack.environment.deactivate() - logfile.close() - continue - - # Concretize environment - print(f"... concretizing ...") - with redirect_stdout(logfile), redirect_stderr(logfile): - with env.write_transaction(): - concretized_specs = env.concretize() - env.write() - spack.environment.display_specs([concrete for _, concrete in concretized_specs]) - - if args.until == "concretize": - spack.environment.deactivate() - logfile.close() - continue - - print("... validating concretization ...") - # Check for duplicate packages - with open(os.path.join(env_dir_full_path, "spack.lock"), "r") as f: - json_to_check = f.read() - ignore_list = [] if "duplicates_to_ignore" not in deployment else deployment["duplicates_to_ignore"] - ret = show_duplicate_packages(json_to_check, ignore_list=ignore_list) - assert ret==0, "Duplicates found! Check spack.lock/show_duplicate_packages.py" - - # Fail if there packages that shouldn't be built with GCC are built with GCC: - all_compilers = set() - for spec in env.all_specs(): - for language in ("c", "cxx", "fortran"): - if language not in spec: continue - compiler_name = spec[language].name - all_compilers.add(compiler_name) - if "allowed_gcc_packages" in deployment: - is_legal = not (compiler_name == "gcc" and spec.name not in deployment["allowed_gcc_packages"]) - assert is_legal, f"spec '{spec.name}/{spec.dag_hash()}' to be built with GCC but not in 'allowed_gcc_packages'!" - - if args.until == "validate": - spack.environment.deactivate() - logfile.close() - continue - - # Fetch packages - print(f"... fetching packages ...") - for spec in env.all_specs(): - logfile.write(f"Fetching {spec.name}@{spec.version}/{spec.dag_hash(length=7)}\n") - with redirect_stdout(logfile), redirect_stderr(logfile): - spec.package.do_fetch() - - if args.until == "fetch": - spack.environment.deactivate() - logfile.close() - continue - - # Install packages - print("... installing", end="") - if deployment["packages_to_install"]: - print(" specs: " +" ".join(deployment["packages_to_install"]), end="") - print(" ...") - if args.no_scheduler: - specs = env.all_matching_specs(*(" ".join(deployment["packages_to_install"]))) - env.install_specs(specs) - else: - logfile.write("Starting install jobs via job scheduler...\n") - if not args.skip_go_rust_handling: - run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, logfile, logfilepath, packages_to_install=["rust", "go"], suffix=".rustgo") - shell_env = os.environ.copy() - shell_env["SPACK_ENV"] = env_dir_full_path - subprocess.run( - os.path.join(spack_stack_dir, "util", "fetch_cargo_deps.py"), - env=shell_env, - stdout=logfile, - stderr=logfile, - check=True, - text=True, - ) - subprocess.run( - os.path.join(spack_stack_dir, "util", "fetch_go_deps.py"), - env=shell_env, - stdout=logfile, - stderr=logfile, - check=True, - text=True, - ) - run_batch_install(deployments_yaml["batch_config"], deployment, env_dir_full_path, logfile, logfilepath, packages_to_install=deployment["packages_to_install"]) - - if args.until == "install": - spack.environment.deactivate() - logfile.close() - continue - - # Generate modules - print(f"... writing package modules ...") - subprocess.run( - ["spack", "--env", env_dir_full_path, "module", "lmod", "refresh", "--yes-to-all", "--upstream-modules"], - stdout=logfile, - stderr=logfile, - check=True, - text=True, - ) - # Also generate a modules dir with a flat structure, i.e., everything is under Core with no metamodules - cfg_hierarchy = "modules:default:lmod:hierarchy::[]" - cfg_compilers = "modules:default:lmod:core_compilers::[%s]" % ",".join(all_compilers) - cfg_root = "modules:default:roots:lmod:$env/modules_flat" - subprocess.run( - [ - "spack", "--env", env_dir_full_path, - "--config", cfg_hierarchy, - "--config", cfg_compilers, - "--config", cfg_root, - "module", "lmod", "refresh", "--yes-to-all" - ] - ) - - # Meta modules - print(f"... writing metamodules ...") - setup_meta_modules() - - # Close this deployment's logfile and zero out spack.environment's stored config info - spack.environment.deactivate() - logfile.close() - - print(f"... done.") From edba38ee625dee2d83bc4f98223203d3abdb9951 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 30 Dec 2025 09:43:13 -0700 Subject: [PATCH 09/12] [release/2.0] Move documentation from readthedocs to Wiki (#1855) * Remove readthedocs documentation --- .gitmodules | 4 - doc/CMakeLists.txt | 30 - doc/CMakeModules | 1 - doc/modulefile_templates/gcc | 23 - doc/modulefile_templates/git-lfs | 15 - doc/modulefile_templates/libfabric | 18 - doc/modulefile_templates/mysql | 19 - doc/modulefile_templates/node.js | 23 - doc/modulefile_templates/openmpi | 37 -- doc/modulefile_templates/qt | 18 - doc/requirements.txt | 1 - doc/source/Acronyms.rst | 31 - doc/source/AddingTestPackages.rst | 31 - doc/source/AutomatedBuilds.rst | 55 -- doc/source/BuildingContainers.rst | 37 -- doc/source/CreatingEnvironments.rst | 39 -- doc/source/Documentation.rst | 26 - doc/source/Environments.rst | 61 -- doc/source/Glossary.rst | 26 - doc/source/KnownIssues.rst | 110 ---- doc/source/MaintainersSection.rst | 435 -------------- doc/source/NewSiteConfigs.rst | 831 -------------------------- doc/source/Overview.rst | 31 - doc/source/PreConfiguredSites.rst | 649 -------------------- doc/source/SpackStackExtension.rst | 40 -- doc/source/UsingSpackEnvironments.rst | 116 ---- doc/source/Utilities.rst | 90 --- doc/source/_templates/.gitignore | 4 - doc/source/conf.py | 174 ------ doc/source/index.rst | 25 - 30 files changed, 3000 deletions(-) delete mode 100644 doc/CMakeLists.txt delete mode 160000 doc/CMakeModules delete mode 100644 doc/modulefile_templates/gcc delete mode 100644 doc/modulefile_templates/git-lfs delete mode 100644 doc/modulefile_templates/libfabric delete mode 100644 doc/modulefile_templates/mysql delete mode 100644 doc/modulefile_templates/node.js delete mode 100644 doc/modulefile_templates/openmpi delete mode 100644 doc/modulefile_templates/qt delete mode 100644 doc/requirements.txt delete mode 100644 doc/source/Acronyms.rst delete mode 100644 doc/source/AddingTestPackages.rst delete mode 100644 doc/source/AutomatedBuilds.rst delete mode 100644 doc/source/BuildingContainers.rst delete mode 100644 doc/source/CreatingEnvironments.rst delete mode 100644 doc/source/Documentation.rst delete mode 100644 doc/source/Environments.rst delete mode 100644 doc/source/Glossary.rst delete mode 100644 doc/source/KnownIssues.rst delete mode 100644 doc/source/MaintainersSection.rst delete mode 100644 doc/source/NewSiteConfigs.rst delete mode 100644 doc/source/Overview.rst delete mode 100644 doc/source/PreConfiguredSites.rst delete mode 100644 doc/source/SpackStackExtension.rst delete mode 100644 doc/source/UsingSpackEnvironments.rst delete mode 100644 doc/source/Utilities.rst delete mode 100644 doc/source/_templates/.gitignore delete mode 100644 doc/source/conf.py delete mode 100644 doc/source/index.rst diff --git a/.gitmodules b/.gitmodules index 303cc47eb..fa7bd2819 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,10 +2,6 @@ path = spack url = https://github.com/jcsda/spack branch = release/2.0 -[submodule "doc/CMakeModules"] - path = doc/CMakeModules - url = https://github.com/noaa-emc/cmakemodules - branch = v1.3.0 [submodule "repos/builtin"] path = repos/builtin url = https://github.com/jcsda/spack-packages diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt deleted file mode 100644 index b738eb9c1..000000000 --- a/doc/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -cmake_minimum_required(VERSION 3.18 FATAL_ERROR) - -project(spack-stack VERSION 2.0.0) - -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/Modules") - -find_package(Sphinx REQUIRED) - -# configured documentation tools and intermediate build results -set(BINARY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/_build") - -# Sphinx cache with pickled ReST documents -set(SPHINX_CACHE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_doctrees") - -# HTML output directory -set(SPHINX_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/html") - -configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/source/conf.py" - "${BINARY_BUILD_DIR}/conf.py" - @ONLY) - -add_custom_target(spack-stack-doc-html ALL - ${SPHINX_EXECUTABLE} - -q -b html - -c "${BINARY_BUILD_DIR}" - -d "${SPHINX_CACHE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/source" - "${SPHINX_HTML_DIR}" - COMMENT "Building HTML documentation with Sphinx") diff --git a/doc/CMakeModules b/doc/CMakeModules deleted file mode 160000 index 607d9b463..000000000 --- a/doc/CMakeModules +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 607d9b4633395b4be88d87a1df88b46302d48d0c diff --git a/doc/modulefile_templates/gcc b/doc/modulefile_templates/gcc deleted file mode 100644 index f6c0f5afd..000000000 --- a/doc/modulefile_templates/gcc +++ /dev/null @@ -1,23 +0,0 @@ -#%Module1.0 - -module-whatis "Provides a gcc-10.1.0 installation for use with spack." - -conflict gnu -conflict gcc - -proc ModulesHelp { } { -puts stderr "Provides a gcc-10.1.0 installation for use with spack." -} - - -# Set this value -set GCC_PATH "/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0" - -prepend-path PATH "${GCC_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib" -prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib64" -prepend-path LIBRARY_PATH "${GCC_PATH}/lib" -prepend-path LIBRARY_PATH "${GCC_PATH}/lib64" -prepend-path CPATH "${GCC_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${GCC_PATH}" -prepend-path MANPATH "${GCC_PATH}/share/man" diff --git a/doc/modulefile_templates/git-lfs b/doc/modulefile_templates/git-lfs deleted file mode 100644 index 3b6c1e097..000000000 --- a/doc/modulefile_templates/git-lfs +++ /dev/null @@ -1,15 +0,0 @@ -#%Module1.0 - -module-whatis "Provides git-lfs-2.10.0 for use with spack." - -conflict git-lfs - -proc ModulesHelp { } { -puts stderr "Provides git-lfs-2.10.0 for use with spack." -} - -# Set this value -set GITLFS_PATH "/my/path/to/spack-stack/git-lfs-2.10.0" - -prepend-path PATH "${GITLFS_PATH}/bin" -prepend-path MANPATH "${GITLFS_PATH}/share/man" diff --git a/doc/modulefile_templates/libfabric b/doc/modulefile_templates/libfabric deleted file mode 100644 index 970294a2b..000000000 --- a/doc/modulefile_templates/libfabric +++ /dev/null @@ -1,18 +0,0 @@ -#%Module1.0 - -module-whatis "Provides a libfabric installation for use with spack." - -conflict libfabric - -proc ModulesHelp { } { -puts stderr "Provides a libfabric installation for use with spack." -} - -# Set this value -set LIBFABRIC_PATH "/opt/cray/libfabric/1.15.2.0" - -prepend-path PATH "${LIBFABRIC_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${LIBFABRIC_PATH}/lib64" -prepend-path LIBRARY_PATH "${LIBFABRIC_PATH}/lib64" -prepend-path CPATH "${LIBFABRIC_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${LIBFABRIC_PATH}" diff --git a/doc/modulefile_templates/mysql b/doc/modulefile_templates/mysql deleted file mode 100644 index 76fdb1646..000000000 --- a/doc/modulefile_templates/mysql +++ /dev/null @@ -1,19 +0,0 @@ -#%Module1.0 - -module-whatis "Provides a mysql-8.0.31 installation for use with spack." - -conflict mysql - -proc ModulesHelp { } { -puts stderr "Provides a mysql-8.0.31 installation for use with spack." -} - -# Set this value -set MYSQL_PATH "/work/noaa/da/role-da/spack-stack/mysql-8.0.31" - -prepend-path PATH "${MYSQL_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${MYSQL_PATH}/lib" -prepend-path LIBRARY_PATH "${MYSQL_PATH}/lib" -prepend-path CPATH "${MYSQL_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${MYSQL_PATH}" -setenv mysql_ROOT "${MYSQL_PATH}" \ No newline at end of file diff --git a/doc/modulefile_templates/node.js b/doc/modulefile_templates/node.js deleted file mode 100644 index 7f5fc113c..000000000 --- a/doc/modulefile_templates/node.js +++ /dev/null @@ -1,23 +0,0 @@ -#%Module1.0 - -module-whatis "Provides an node.js-20.10.0 installation for use with spack." - -# Only allow one instance of compiler to load -conflict ecflow - -proc ModulesHelp { } { -puts stderr "Provides an node.js-20.10.0 installation for use with spack." -} - -if { [ module-info mode load ] && ![ is-loaded gcc/10.3.0 ] } { - module load gcc/10.3.0 -} - -# Set this value -set NODEJS_PATH "/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0" - -prepend-path PATH "${NODEJS_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${NODEJS_PATH}/lib" -prepend-path LIBRARY_PATH "${NODEJS_PATH}/lib" -prepend-path CPATH "${NODEJS_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${NODEJS_PATH}" diff --git a/doc/modulefile_templates/openmpi b/doc/modulefile_templates/openmpi deleted file mode 100644 index b13398bbb..000000000 --- a/doc/modulefile_templates/openmpi +++ /dev/null @@ -1,37 +0,0 @@ -#%Module1.0 - -module-whatis "Provides an openmpi-4.1.3 installation for use with spack and gcc-10.1.0." - -conflict openmpi -conflict mpich -conflict mpi - -proc ModulesHelp { } { -puts stderr "Provides an openmpi-4.1.3 installation for use with spack and gcc-10.1.0." -} - -if { [ module-info mode load ] && ![ is-loaded comp/gcc/10.1.0 ] } { - module load comp/gcc/10.1.0 -} - -# Set this value -set OPENMPI_PATH "/discover/swdev/jcsda/spack-stack/openmpi-4.1.3/gcc-10.1.0" - -prepend-path PATH "${OPENMPI_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${OPENMPI_PATH}/lib" -prepend-path LIBRARY_PATH "${OPENMPI_PATH}/lib" -prepend-path CPATH "${OPENMPI_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${OPENMPI_PATH}" -prepend-path MANPATH "${OPENMPI_PATH}/share/man" - -# Settings specific for Discover -setenv MPIHOME ${OPENMPI_PATH} -setenv MPI_HOME ${OPENMPI_PATH} -unsetenv SLURM_EXPORT_ENV -setenv PSM2_PATH_SELECTION "static_base" -setenv SLURM_CPU_BIND "none" - -# Settings specific for Cheyenne and Hercules -setenv MPI_ROOT ${OPENMPI_PATH} -setenv UCX_MAX_RNDV_RAILS "1" -setenv OMPI_MCA_btl "^openib" diff --git a/doc/modulefile_templates/qt b/doc/modulefile_templates/qt deleted file mode 100644 index 9ba14fbc5..000000000 --- a/doc/modulefile_templates/qt +++ /dev/null @@ -1,18 +0,0 @@ -#%Module1.0 - -module-whatis "Provides a qt-5.15.2 installation for use with spack." - -conflict qt - -proc ModulesHelp { } { -puts stderr "Provides a qt-5.15.2 installation for use with spack." -} - -# Set this value -set QT_PATH "/discover/swdev/jcsda/spack-stack/qt-5.15.2/5.15.2/gcc_64" - -prepend-path PATH "${QT_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${QT_PATH}/lib" -prepend-path LIBRARY_PATH "${QT_PATH}/lib" -prepend-path CPATH "${QT_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${QT_PATH}" diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index e4d7905c5..000000000 --- a/doc/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -urllib3<2 diff --git a/doc/source/Acronyms.rst b/doc/source/Acronyms.rst deleted file mode 100644 index 8ef55c114..000000000 --- a/doc/source/Acronyms.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _Acronyms: - -Acronyms -************************* - -.. table:: - :widths: 20 80 - - +----------------+---------------------------------------------------+ - | Abbreviation | Explanation | - +================+===================================================+ - | ESMF | The Earth System Modeling Framework | - +----------------+---------------------------------------------------+ - | FMS | Flexible Modeling System | - +----------------+---------------------------------------------------+ - | HTML | Hypertext Markup Language | - +----------------+---------------------------------------------------+ - | JEDI | Joint Effort for Data assimilation Integration | - +----------------+---------------------------------------------------+ - | JCSDA | Joint Center for Satellite Data Assimilation | - +----------------+---------------------------------------------------+ - | MPI | Message Passing Interface | - +----------------+---------------------------------------------------+ - | NOAA | National Oceanic and Atmospheric Administration | - +----------------+---------------------------------------------------+ - | OpenMP | Open Multi-Processing | - +----------------+---------------------------------------------------+ - | UCAR | University Corporation for Atmospheric Research | - +----------------+---------------------------------------------------+ - | UFS | Unified Forecast System | - +----------------+---------------------------------------------------+ diff --git a/doc/source/AddingTestPackages.rst b/doc/source/AddingTestPackages.rst deleted file mode 100644 index 991b85962..000000000 --- a/doc/source/AddingTestPackages.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _Add_Test_Packages: - -Adding test packages (chaining environments) -******************************************** - -Releases of spack-stack are deployed quarterly on supported platforms. Between releases, additional packages may be installed through the following steps, which make use of Spack's environment chaining capabilities. This allows parts of the stack to be replaced while leaving the release environment untouched. - -To install in an additional environment within an official spack-stack space, simply ``cd`` into the appropriate spack-stack root directory and run ``. setup.sh`` before continuing to the following steps. To create a chained environment in a personal space outside of an official installation, it is recommended that you use the same spack-stack release as the one from which you wish to use an upstream environment. For instance, if you are targeting an environment installed under ``spack-stack-1.4.1/``, then use ``git clone --recurse-submodules https://github.com/jcsda/spack-stack -b release/1.4.1`` to set up your installation, then in the newly created spack-stack directory run ``. setup.sh`` and proceed with the following steps. The following steps install a hypothetical new version of NetCDF C on Hera that also rebuilds all of its dependencies for the UFS Weather Model. Note that it is recommended to use one or more `meta-modules `_ (e.g., 'ufs-weather-model-env') in the spec as opposed to only providing specific packages, as using the meta-modules will ensure that all of the module files needed from the upstream environment for a given application are copied into the downstream environment by the module refresh command. Make sure to use the same compiler as the upstream environment. - -.. code-block:: bash - - spack stack create env --name netcdf-test --template empty --site hera --compiler gcc \ - --upstream /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ue-gcc/install [--upstream /path/to/second/install] [--modify-pkg netcdf-c] - cd envs/netcdf-test - spack env activate . - spack add ufs-weather-model-env%intel ^netcdf-c@5.0.0 - spack concretize 2>&1 | tee log.concretize - spack install 2>&1 | tee log.install - spack module lmod refresh --upstream-modules - spack stack setup-meta-modules - -To use the environment, access it in the same way as a regular spack-stack installation, i.e., add the directory provided by ``spack stack setup-meta-modules`` ending with '/modulefiles/Core' to your MODULEPATH variable. This will give access to the upstream modules needed while avoiding package conflicts; do not use the upstream environment in ``$MODULEPATH`` directly. - -.. note:: - The ``--upstream`` option for the ``spack stack create env`` command adds a specified Spack/spack-stack installation path as an upstream environment in the resulting spack.yaml, and can be invoked multiple times in order to include multiple upstream environments. The command will *give a warning but not fail* if an invalid directory is provided (either because it does not end with the typical ``install`` directory name, or because it does not exist). Be mindful of these warnings, and if the path does not exist, check for typos and make sure that you are using the path for the correct system from the table in :numref:`Section %s `. - -.. note:: - The ``--modify-pkg`` option for the ``spack stack create env`` command should be used by spack-stack maintainers whenever a package recipe needs to be modified for between-release deployments (i.e., chained environments that live in an official spack-stack release), such as when a new version, variant, or patch needs to be added. This option creates a separate, custom Spack repo specific to the environment being created, and lives under that environment's directory structure (i.e., ``$SPACK_ENV/envrepo/``). It requires the name of the package to be customized, and can be invoked multiple times to customize multiple packages. Each package specified will be copied into the custom repo, and the environment's repo configuration will be automatically updated to use those custom copies. It is important to use this option in order to avoid contaminating or breaking the existing installation. - -.. note:: - More details on chaining spack environments and a few words of caution can be found in the `Spack documentation `_. Those words of caution need to be taken seriously, especially those referring to not deleting modulefiles and dependencies in the upstream spack environment (if having permissions to do so)! diff --git a/doc/source/AutomatedBuilds.rst b/doc/source/AutomatedBuilds.rst deleted file mode 100644 index c901c3ebe..000000000 --- a/doc/source/AutomatedBuilds.rst +++ /dev/null @@ -1,55 +0,0 @@ -================ -Automated builds -================ - -.. _AutomatedBuildWorkflow: - -Workflow -************************* - -The spack-stack repository now contains scripts for implementing scheduled, automated building, unit testing, and build caching of packages. These scripts live under utils/weekly_build/. They download their own copy of spack-stack (the develop branch, by default) and build one or more environments under that directory structure, which may be transient or reused. - -Script failures are detected by non-zero exit codes. - -The list of packages to be unit tested is stored in the ``$PACKAGES_TO_TEST`` variable in ShellSetup.sh. As unit testing needs are identified, this list should be updated to be as comprehensive as possible based on https://github.com/JCSDA/spack-stack/wiki/Automated-testing-and-building#unit-testing-requirements. - -.. _AutomatedBuildAddPlatform: - -Overview of implementing on a new platform -****************************************** - -Use a clean environment. Do NOT run setup.sh in the directory containing the build scripts (util/weekly_build/) to be run. Nothing in spack-stack outside of the build scripts is needed to get the automated build workflow running, as the workflow will take care of download a copy of spack-stack to be used for doing the automated builds themselves. - -------------------------------- -Platform-specific configuration -------------------------------- - -The ShellSetup.sh script points to a script in the sites/ subdirectory based on the site name as set by the third argument of each build script (stored and used as ``$PLATFORM`` throughout the scripts). Note that a number of variables and functions can be set or overridden here: - - ``$COMPILERS``: Space-delimited list of compilers to be built against on a given system. - - ``$TEMPLATES``: Space-delimited list of templates to install. Default is 'unified-env'. - - ``$PACKAGES_TO_TEST``: Space-delimited list of packages for which units tests should be run. - - ``$PACKAGES_TO_INSTALL``: Space-delimited list of packages to be installed. For many systems it can be left empty so as to install the entire Unified Environment. - - ``alert_cmd()``: This function should be set for each system. It is invoked when any step fails (non-zero exit code somewhere in the workflow), and so can be used to send emails, post to Slack or GitHub, etc. - - ``spack_wrapper()``: This function runs the Spack command, where the first argument is a log file name. It should not typically need to be redefined by platform. - - ``spack_install_wrapper()``: This function determines the means by which the ``spack`` command is invoked for the install steps, and can be overridden for each platform. If testing and installation will be performed outside of a batch scheduler, i.e., directly on a login node, then the default can probably be used. However, overriding this function allows, for instance, for job schedulers to be invoked (srun, qsub, etc.), as well as for parallel installations to be performed. Note that the first argument is a log file name, so custom versions of this function will most likely need to use ``shift`` to properly access the remaining arguments. - - ``$INSTALL_OPTS``: Additional ``spack install`` flags to be supplied to ``spack_install_wrapper()``. - - ``$TEST_``: Set this variable to ON in order to enable the corresponding test under util/weekly_build/apptests/. - - ``$BUILD_CACHE``: Name of or path to (beginning with '\file://') build cache directory. Defaults to 'local-binary' mirror defined in site's mirrors.yaml. - - ``$SOURCE_CACHE``: Name of or path to (beginning with '\file://') source cache directory. Defaults to 'local-source' mirror defined in site's mirrors.yaml. - - ``$PADDED_LENGTH``: Padded length setting for Spack build cache generation. It should be as long as possible without the build failing. - - ``$KEEP_WEEKLY_BUILD_DIR``: Set to 'YES' to use a persistent directory structure for automated builds. - - ``$DISABLE_TESTING``: Set to 'YES' to switch off build-time package testing regardless of whether ``$PACKAGES_TO_TEST`` is set. - ------------------------------- -Setting up weekly builds: cron ------------------------------- - -Prior to running, obtain a current copy of the build scripts by cloning the spack-stack repository to the platform of interest (/path/to/spack-stack-auto-build/). - -If /path/to/spack-stack is the local spack-stack root directory containing spack-stack-x.y.z installations, run ``mkdir -p /path/to/spack-stack/weekly_build/logs``. - -In crontab, add: - -.. code-block:: console - - 0 1 * * SUN /path/to/spack-stack-auto-build/util/weekly_build/SpackStackBuildCache_AllSteps.sh $(date +\%y\%m\%d) /path/to/spack-stack/weekly_build acorn > /path/to/spack-stack/weekly_build/logs/cron.$(date +\%y\%m\%d).out 2>&1 diff --git a/doc/source/BuildingContainers.rst b/doc/source/BuildingContainers.rst deleted file mode 100644 index 52da0326f..000000000 --- a/doc/source/BuildingContainers.rst +++ /dev/null @@ -1,37 +0,0 @@ -.. _BuildingContainers: - - -Creating Containers -******************** - -================= -Container recipes -================= - -Container recipes are self-contained except that at container creation time (``spack stack create ctr ...``) package information from the common ``packages.yaml`` (or a manually provided version) is added to the ``packages:`` section, and specs are added in the section ``specs`` from the specs file provided as an argument. Container recipes are defined in ``configs/containers``, and specs are defined in ``configs/containers/specs``. - -**WORK IN PROGRESS** - - ------------------------ -Container build example ------------------------ - -It is important to know that container builds do not allow for multiple versions of the same package (e.g., ``fms@2022.01`` and ``fms@release-jcsda``), which needs to be taken into account when creating the specs in ``configs/containers/specs/*.yaml``. - -.. code-block:: console - - # See a list of preconfigured containers - spack stack create ctr -h - - # Create container spack definition (spack.yaml) in directory envs/ - spack stack create ctr --container=docker-ubuntu-gcc-openmpi --specs=jedi-ci - - # Descend into container environment directory - cd envs/docker-ubuntu-gcc-openmpi - - # Docker: create Dockerfile and build container - # See section "container" in spack.yaml for additional information - spack containerize > Dockerfile - docker build -t myimage . - docker run -it myimage diff --git a/doc/source/CreatingEnvironments.rst b/doc/source/CreatingEnvironments.rst deleted file mode 100644 index fc02c289f..000000000 --- a/doc/source/CreatingEnvironments.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. _CreatingEnvironment: - -Creating and installing new environments -**************************************** - -The following instructions are a simplified version of the instructions found in :numref:`Sections %s ` and :numref:`%s `, and should work for systems where spack-stack already has a site configuration, or where minimal configuration is necessary. For more detailed instructions, including how to generate new site configurations and more details of when and how to use external packages, see :numref:`Sections %s ` and :numref:`%s `. To chain a new Spack environment to an existing one, such as to test a new package version based on dependencies already installed in the upstream environment, see :numref:`%s `. - -Note that items in "<>" should be replaced with the appropriate values (site names, etc.), and items in '[]' are optional. ** and *