diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index b4181e62..b2a03985 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -67,20 +67,20 @@ jobs: DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 HOST_ARCH: ppc64le RUNS_ON: streak2 - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "%gcc@10.1.0~pygeosx~openmp" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - RUNS_ON: ubuntu-latest - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 - TPL_DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 - SPEC: "%gcc@10.1.0~pygeosx~openmp cuda_arch=70 ^cuda@12.4.0+allow-unsupported-compilers" - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - RUNS_ON: ubuntu-latest + # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "~pygeosx~openmp %gcc@10.1.0" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + # RUNS_ON: ubuntu-latest + # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 12.4.0, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11 + # TPL_DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda12.4.0-openblas0.3.10-zlib1.2.11-cmake3.31.4-no-geosx:0.0.1 + # SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + # RUNS_ON: ubuntu-latest steps: - name: Checkout diff --git a/.gitignore b/.gitignore index 228b4823..704d7cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .vscode/ spack-*.txt __pycache__ +uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index a50db3f9..1bee4505 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,7 +4,8 @@ "package_final_phase": "lvarray_hostconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2bfcc69fa870d3c6919be87593f22647981b648a", +"spack_commit": "734c5db2121b01c373eed6538e452f18887e9e44", "spack_configs_path": "scripts/spack_configs", -"spack_packages_path": "scripts/spack_packages/packages" +"spack_packages_path": "scripts/spack_packages/packages", +"spack_packages_commit": "a75a7f75182ffc7a51c6ca7f0fec4bf9b2705be8" } diff --git a/docker/TotalEnergies/Pangea3.Dockerfile b/docker/TotalEnergies/Pangea3.Dockerfile index 61434384..3d67d82e 100644 --- a/docker/TotalEnergies/Pangea3.Dockerfile +++ b/docker/TotalEnergies/Pangea3.Dockerfile @@ -48,9 +48,11 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ ln -s /usr/lib64/libnsl.so.2.0.0 /usr/lib64/libnsl.so && \ ln -s /usr/lib64/librdmacm.so.1.3.48.0 /usr/lib64/librdmacm.so && \ ln -s /usr/lib64/liblustreapi.so.1 /usr/lib64/liblustreapi.so && \ + ln -s /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib/libbat.so /usr/lib64/libbat.so && \ + ln -s /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib/liblsf.so /usr/lib64/liblsf.so && \ # Run uberenv ./scripts/uberenv/uberenv.py \ - --spec "%gcc@9.4.0+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %gcc-9 ^cuda@11.5.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/pangea-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -61,7 +63,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated-wave-solver-only.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Build only the wave solver for Pangea 3 RUN echo 'set ( GEOS_ENABLE_CONTACT OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ diff --git a/docker/pangea-spack.yaml b/docker/pangea-spack.yaml index a505347d..274a1dce 100644 --- a/docker/pangea-spack.yaml +++ b/docker/pangea-spack.yaml @@ -17,31 +17,41 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - spec: gcc@=9.4.0 - paths: - cc: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gcc - cxx: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/g++ - f77: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran - fc: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran - flags: {} - operating_system: almalinux8 - target: ppc64le - modules: [] - environment: - prepend_path: - LD_LIBRARY_PATH: /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib - extra_rpaths: [] + toolchains: + gcc-9: + - spec: '%c=gcc@9.4.0' + when: '%c' + - spec: '%cxx=gcc@9.4.0' + when: '%cxx' + - spec: '%fortran=gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' packages: all: target: [x86_64] - compiler: [gcc] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] + + mpi: + require: + - openmpi + + blas: + require: + - "openblas" + lapack: + require: + - "openblas" + + gcc: + externals: + - spec: gcc@9.4.0 languages:='c,c++,fortran' + prefix: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v + extra_attributes: + compilers: + c: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gcc + cxx: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/g++ + fortran: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran autoconf: externals: @@ -94,10 +104,6 @@ spack: - spec: openblas@0.3.18 prefix: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-9.4.0/openblas-0.3.18-vk36pzksytuhylqesg4cca7667np5sjp buildable: false - blas: - buildable: false - lapack: - buildable: false libtool: externals: @@ -110,13 +116,9 @@ spack: prefix: /usr buildable: false - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - openmpi: externals: - - spec: openmpi@4.1.2%gcc@9.4.0 +cuda~cxx~cxx_exceptions~java+lustre~memchecker+pmi+static~wrapper-rpath schedulers=lsf + - spec: openmpi@4.1.2 +cuda~cxx~cxx_exceptions~java+lustre~memchecker+pmi+static~wrapper-rpath schedulers=lsf %gcc@9.4.0 prefix: /data_local/sw/openmpi/4.1.2/env/gcc-8.4.1 # Built-in ncurses causes spack failure @@ -137,7 +139,7 @@ spack: buildable: false python: externals: - - spec: python@3.8.17+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib + - spec: python@3.8.17+bz2+crypt+ctypes+dbm+lzma+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib prefix: /usr buildable: false readline: diff --git a/docker/rocky-spack.yaml b/docker/rocky-spack.yaml index 94025319..4a8c51bd 100644 --- a/docker/rocky-spack.yaml +++ b/docker/rocky-spack.yaml @@ -17,44 +17,62 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: rocky8 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - spec: clang@17.0.6 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: rocky8 - paths: - cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ - f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran - spec: gcc@13.3.1 - target: x86_64 + toolchains: + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%openmpi@4.1.1' + when: '%mpi' + clang-17: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%[virtuals=c]llvm@17.0.6+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@17.0.6+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@13.3.1' + when: '%fortran' + - spec: '%openmpi@4.1.1' + when: '%mpi' packages: all: target: [x86_64] - compiler: [gcc, clang] - providers: - blas: [netlib-lapack] - lapack: [netlib-lapack] - mpi: [openmpi] + + mpi: + require: + - openmpi + + blas: + require: + - "netlib-lapack" + lapack: + require: + - "netlib-lapack" + + llvm: + externals: + - spec: llvm@17.0.6+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + gcc: + externals: + - spec: gcc@13.3.1 languages:='c,c++,fortran' + prefix: /opt/rh/gcc-toolset-13/root/usr + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran autoconf: version: [2.71] @@ -77,9 +95,9 @@ spack: cuda: buildable: False externals: - - spec: cuda@12.9.1%gcc@13.3.1 +allow-unsupported-compilers + - spec: cuda@12.9.1 +allow-unsupported-compilers %gcc@13.3.1 prefix: /usr/local/cuda - - spec: cuda@12.9.1%clang@17.0.6 +allow-unsupported-compilers + - spec: cuda@12.9.1 +allow-unsupported-compilers %llvm@17.0.6 prefix: /usr/local/cuda m4: buildable: false @@ -87,10 +105,6 @@ spack: - spec: m4@1.4.18 prefix: /usr - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - netlib-lapack: buildable: false externals: diff --git a/docker/spack.yaml b/docker/spack.yaml index 463e8ead..1a977a5c 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -17,111 +17,130 @@ spack: - ../defaults.yaml - ../versions.yaml - - compilers:: - - compiler: - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu22.04 - paths: - cc: /usr/bin/clang-15 - cxx: /usr/bin/clang++-15 - f77: /usr/bin/gfortran-11 - fc: /usr/bin/gfortran-11 - spec: clang@15 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: {} - modules: [] - operating_system: ubuntu20.04 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - spec: clang@10 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu22.04 - paths: - cc: /usr/bin/gcc-12 - cxx: /usr/bin/g++-12 - f77: /usr/bin/gfortran-12 - fc: /usr/bin/gfortran-12 - spec: gcc@12 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu22.04 - paths: - cc: /usr/bin/gcc-11 - cxx: /usr/bin/g++-11 - f77: /usr/bin/gfortran-11 - fc: /usr/bin/gfortran-11 - spec: gcc@11 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu20.04 - paths: - cc: /usr/bin/gcc-10 - cxx: /usr/bin/g++-10 - f77: /usr/bin/gfortran-10 - fc: /usr/bin/gfortran-10 - spec: gcc@10 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: ubuntu20.04 - paths: - cc: /usr/bin/gcc-9 - cxx: /usr/bin/g++-9 - f77: /usr/bin/gfortran-9 - fc: /usr/bin/gfortran-9 - spec: gcc@9 - target: x86_64 - - compiler: - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: centos7 - paths: - cc: /opt/rh/devtoolset-8/root/usr/bin/gcc - cxx: /opt/rh/devtoolset-8/root/usr/bin/g++ - f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran - fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran - spec: gcc@8 - target: x86_64 + toolchains: + clang-15: + - spec: '%[virtuals=c]llvm@15.0.7+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@15.0.7+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + clang-10: + - spec: '%[virtuals=c]llvm@10.0.0+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@10.0.0+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@9.5.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-9: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@9.5.0' + when: '%c' + - spec: '%cxx=gcc@9.5.0' + when: '%cxx' + - spec: '%fortran=gcc@9.5.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-10: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@10.5.0' + when: '%c' + - spec: '%cxx=gcc@10.5.0' + when: '%cxx' + - spec: '%fortran=gcc@10.5.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-11: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@11.4.0' + when: '%c' + - spec: '%cxx=gcc@11.4.0' + when: '%cxx' + - spec: '%fortran=gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' + gcc-12: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@12.3.0' + when: '%c' + - spec: '%cxx=gcc@12.3.0' + when: '%cxx' + - spec: '%fortran=gcc@12.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2' + when: '%mpi' packages: all: target: [x86_64] - compiler: [gcc, clang] - providers: - blas: [netlib-lapack] - lapack: [netlib-lapack] - mpi: [openmpi] + + mpi: + require: + - openmpi + + blas: + require: + - "netlib-lapack" + lapack: + require: + - "netlib-lapack" + + llvm: + externals: + - spec: llvm@10.0.0+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + - spec: llvm@15.0.7+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-15 + cxx: /usr/bin/clang++-15 + + gcc: + externals: + - spec: gcc@9.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-9 + cxx: /usr/bin/g++-9 + fortran: /usr/bin/gfortran-9 + - spec: gcc@10.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-10 + cxx: /usr/bin/g++-10 + fortran: /usr/bin/gfortran-10 + - spec: gcc@11.4.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-11 + cxx: /usr/bin/g++-11 + fortran: /usr/bin/gfortran-11 + - spec: gcc@12.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 autoconf: version: [2.71] @@ -163,10 +182,6 @@ spack: - spec: mpfr@6.0.2 prefix: /usr - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - netlib-lapack: buildable: false externals: @@ -179,18 +194,16 @@ spack: prefix: /usr openmpi: externals: - - spec: openmpi@4.1.2%clang + - spec: openmpi@4.1.2 %llvm prefix: /usr - - spec: openmpi@1.10.7%gcc@8 - prefix: /usr/lib64/openmpi modules: [mpi] - - spec: openmpi@4.1.2%gcc@9 + - spec: openmpi@4.1.2 %gcc@9.5.0 prefix: /usr - - spec: openmpi@4.1.2%gcc@10 + - spec: openmpi@4.1.2 %gcc@10.5.0 prefix: /usr - - spec: openmpi@4.1.2%gcc@11 + - spec: openmpi@4.1.2 %gcc@11.4.0 prefix: /usr - - spec: openmpi@4.1.2%gcc@12 + - spec: openmpi@4.1.2 %gcc@12.3.0 prefix: /usr perl: buildable: false diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index bffe8879..f761368c 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -15,7 +15,7 @@ RUN dnf clean all && \ dnf -y install \ which \ clang-17.0.6 \ - gcc-gfortran \ + gcc-toolset-13 \ python3 \ zlib-devel \ tbb \ @@ -70,7 +70,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so && \ ./scripts/uberenv/uberenv.py \ - --spec "%clang@17.0.6+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %clang-17 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/rocky-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -81,7 +81,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPL's from the previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index faf56712..d73e87ff 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -67,7 +67,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ ln -s /usr/lib64/libblas.so.3 /usr/lib64/libblas.so && \ ln -s /usr/lib64/liblapack.so.3 /usr/lib64/liblapack.so && \ ./scripts/uberenv/uberenv.py \ - --spec "%gcc@13.3.1+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/rocky-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -78,7 +78,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPL's from the previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index 0fe92b43..1a1b3628 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get install -y --no-install-recommends \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "%clang@10+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "+cuda~uncrustify~openmp~pygeosx cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -73,7 +73,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPL's from previous stage diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 55094101..4e0058ad 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -58,6 +58,7 @@ ARG GCC_MAJOR_VERSION RUN apt-get install -y --no-install-recommends \ gfortran-$GCC_MAJOR_VERSION \ + g++-$GCC_MAJOR_VERSION \ libtbb-dev \ make \ bc \ @@ -68,13 +69,26 @@ RUN apt-get install -y --no-install-recommends \ ca-certificates \ git +# Add MPI environment path info +ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ + CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun +# The multi-line definition of arguments does not seem happy +# when a variable uses the value of another variable previously defined on the same line. +ENV OMPI_CC=$CC \ + OMPI_CXX=$CXX + # Run uberenv # Have to create install directory first for uberenv # -k flag is to ignore SSL errors RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ +# Create symlinks to g++ libraries + ln -s /usr/bin/g++-${GCC_MAJOR_VERSION} /usr/bin/g++ && \ ./scripts/uberenv/uberenv.py \ - --spec "%clang@${CLANG_MAJOR_VERSION} ~shared~openmp+docs ^caliper~gotcha~sampler~libunwind~libdw~papi" \ + --spec "~shared~openmp+docs %clang-${CLANG_MAJOR_VERSION} ^caliper~gotcha~sampler~libunwind~libdw~papi" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -85,7 +99,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Extract only TPLs from previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index 736ca370..2f74d46c 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -111,7 +111,7 @@ RUN apt-get install -y --no-install-recommends \ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ ./scripts/uberenv/uberenv.py \ - --spec "%gcc@${GCC_MAJOR_VERSION} ~pygeosx +docs" \ + --spec "~pygeosx +docs %gcc-${GCC_MAJOR_VERSION}" \ --spack-env-file=${SRC_DIR}/docker/spack.yaml \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ @@ -122,7 +122,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ cp *.cmake /spack-generated.cmake && \ # Remove extraneous spack files cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ # Last step is setting everything for a complete slave that will build GEOSX. FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index 8bf5feeb..6ee12ca2 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -1,56 +1,99 @@ #!/bin/bash ## Builds the TPLs for a specific system and host config. -## Usage ./setupLC-TPL-uberenv-helper.bash pathToInstallDirectory machine compiler spackSpecToBuild commandToGetANode +## Usage ./setupLC-TPL-uberenv-helper.bash [ExtraArgs...] + +# --- 1. Initialize Control Variables --- +# By default, we will set permissions and reuse a previous build if available +SET_PERMISSIONS=true +CLEAN=false +: ${USER:=$(whoami)} + +# --- Argument Parsing --- INSTALL_DIR=$1 MACHINE=$2 COMPILER=$3 SPEC=\"${4}\" GET_A_NODE=$5 -## Eat up the command line arguments so the rest can be forwarded to config-build. -shift -shift -shift -shift -shift +# Eat up the primary arguments; the rest in $@ are forwarded. +shift 5 +# --- 2. Check for the --no-permissions Flag --- +# Loop through the forwarded arguments to find our flag. +for arg in "$@"; do + if [[ "$arg" == "--no-permissions" ]]; then + SET_PERMISSIONS=false + echo "Found --no-permissions flag in uberenv-helper. Will skip permission updates." + shift + elif [[ "$arg" == "--clean" ]]; then + CLEAN=true + echo "Found --clean flag in uberenv-helper. Will clean build first" + shift + fi +done + +# --- Main Execution --- CONFIG=$MACHINE-$COMPILER LOG_FILE=$CONFIG.log +# --- Clean build --- +if [ "$CLEAN" = true ]; then + DEST=${INSTALL_DIR}/${CONFIG}_tpls + echo "Removing ${DEST}" && rm -rf "${DEST}" + + DEST=${LOG_FILE} + echo "Removing ${DEST}" && rm -rf "${DEST}" +fi + echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." -ssh $MACHINE -t " -. /etc/profile && -cd $PWD && -$GET_A_NODE ./scripts/uberenv/uberenv.py --spec ${SPEC} --prefix ${INSTALL_DIR}/${CONFIG}_tpls --spack-env-name ${CONFIG}_env && -exit" > $LOG_FILE 2>&1 +# Execute the command to be run on the compute node via ssh +ssh -t "${USER}@${MACHINE}.llnl.gov" " + source /etc/profile && + cd '${PWD}' && + ${GET_A_NODE} bash -c ' + echo \"Start time: \$(date)\" + ./scripts/uberenv/uberenv.py \ + --spec ${SPEC} \ + --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" \ + --spack-env-name \"${CONFIG}_env\" \ + \"\$@\" + echo \"End time: \$(date)\" + ' +" > "${LOG_FILE}" 2>&1 ## Check the last ten lines of the log file. ## A successful install should show up on one of the final lines. -tail -10 $LOG_FILE | grep -E "Successfully installed geos" > /dev/null +tail -10 "$LOG_FILE" | grep -E "Successfully installed geos" > /dev/null if [ $? -eq 0 ]; then - echo "Cleanup extra build files at ${INSTALL_DIR}/${CONFIG}_tpls/ ." - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/${CONFIG}_env - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/.spack-db - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/misc_cache - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/spack - rm -rf ${INSTALL_DIR}/${CONFIG}_tpls/build_stage - - echo "Updating file permissions at ${INSTALL_DIR}/${CONFIG}_tpls/ ." + echo "Cleanup extra build files at ${INSTALL_DIR}/${CONFIG}_tpls/." + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/${CONFIG}_env" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/.spack-db" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/misc_cache" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/spack" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/builtin_spack_packages_repo" + rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/build_stage" + + # --- 3. Conditionally Set Permissions --- + if [ "$SET_PERMISSIONS" = true ]; then + echo "Updating file permissions at ${INSTALL_DIR}/${CONFIG}_tpls/." # Install directory root - chmod g+rx $INSTALL_DIR - chgrp GEOS $INSTALL_DIR + chmod g+rx "$INSTALL_DIR" + chgrp GEOS "$INSTALL_DIR" # Update only executable and library directories to avoid NFS errors - chmod g+rx -R $INSTALL_DIR/${CONFIG}_tpls/bin - chgrp GEOS -R $INSTALL_DIR/${CONFIG}_tpls/bin - chmod g+rx -R $INSTALL_DIR/${CONFIG}_tpls/${COMPILER%%-*}* - chgrp GEOS -R $INSTALL_DIR/${CONFIG}_tpls/${COMPILER%%-*}* + chmod g+rx -R "${INSTALL_DIR}/${CONFIG}_tpls/bin" + chgrp GEOS -R "${INSTALL_DIR}/${CONFIG}_tpls/bin" + chmod g+rx -R "${INSTALL_DIR}/${CONFIG}_tpls/${COMPILER%%-*}"* + chgrp GEOS -R "${INSTALL_DIR}/${CONFIG}_tpls/${COMPILER%%-*}"* + else + echo "Skipping permission updates as requested." + fi - echo "Build of ${CONFIG} completed successfully." - exit 0 + echo "Build of ${CONFIG} completed successfully." + exit 0 else - echo "Build of ${CONFIG} seemed to fail, check $LOG_FILE." - exit 1 + echo "Build of ${CONFIG} seemed to fail, check $LOG_FILE." + exit 1 fi diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index c118b53e..50c5d4c2 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -1,62 +1,147 @@ #!/bin/bash -## Builds the TPLs on all LC systems. Must be run from the top level TPL directory. -## Usage ./setupLC-TPL-uberenv.bash pathToInstallDirectory +## Builds the TPLs on specified LC systems. +## Usage: ./setupLC-TPL-uberenv.bash [MachineList] [ExtraArgs...] +## +## InstallDir: Absolute path to the installation directory. +## MachineList: (Optional) Comma-separated list of machines to build on +## (e.g., "dane,matrix"). Defaults to all. +## ExtraArgs: (Optional) Additional arguments forwarded to the helper script. +## Use --no-permissions to skip all chmod/chgrp calls. +## Use --clean to clean data from previous build. + +# --- Configuration --- +# All known machines. Add new machine names here. +declare -a ALL_MACHINES=("dane" "matrix" "tuolumne") + +# --- Argument Parsing --- INSTALL_DIR=$1 +MACHINE_LIST_STR=${2:-"all"} # Default to "all" if the second argument is not provided -## Eat up the command line arguments so the rest can be forwarded to setupLC-TPL-helper. -shift -shift +# Validate INSTALL_DIR before proceeding +if [[ -z "$INSTALL_DIR" ]]; then + echo "ERROR: No installation directory path was provided." >&2 + exit 1 +fi +if [[ ! -d "$INSTALL_DIR" ]]; then + echo "ERROR: Installation directory '$INSTALL_DIR' does not exist." >&2 + exit 1 +fi +if [[ ! "$INSTALL_DIR" = /* ]]; then + echo "ERROR: Installation directory must be an absolute path." >&2 + exit 1 +fi -## Trap the interupt signal and kill all children. -trap 'killall' INT +# Eat up the first two arguments so the rest can be forwarded. +shift 2 2>/dev/null -killall() { - trap '' INT TERM # ignore INT and TERM while shutting down - echo "**** Shutting down. Killing chid processes ****" # added double quotes - kill -TERM 0 # fixed order, send TERM not INT - wait - echo DONE -} +# --- Initialize Control Variable and Parse Extra Arguments --- +SET_PERMISSIONS=true +declare -a FORWARDED_ARGS=() + +# Loop through remaining args, filter out our flag, and collect the rest. +for arg in "$@"; do + if [[ "$arg" == "--no-permissions" ]]; then + SET_PERMISSIONS=false + fi + FORWARDED_ARGS+=("$arg") +done +# --- Setup --- +# Check for uberenv script if [[ ! -e "scripts/uberenv/uberenv.py" ]]; then - echo "uberenv.py script not found. Please initialize uberenv submodule first." - exit + echo "ERROR: uberenv.py script not found. Please initialize uberenv submodule first." >&2 + exit 1 fi -if [[ -z $INSTALL_DIR ]]; then - echo "No installation directory path was provided" - exit +# Determine which machines to run on +declare -a MACHINES_TO_RUN +if [[ "$MACHINE_LIST_STR" == "all" ]]; then + MACHINES_TO_RUN=("${ALL_MACHINES[@]}") +else + # Convert comma-separated string to array + IFS=',' read -r -a MACHINES_TO_RUN <<< "$MACHINE_LIST_STR" fi -if [[ ! -d $INSTALL_DIR ]]; then - echo "Installation directory $INSTALL_DIR does not exist. Please initialize first." - exit -fi +# --- Functions --- +# Trap the interrupt signal and kill all children. +trap 'kill_children' INT -if [[ ! "$INSTALL_DIR" = /* ]]; then - echo "Installation directory $INSTALL_DIR must be an absolute path." - exit -fi +function kill_children() { + trap '' INT TERM # Ignore signals while shutting down + echo -e "\n**** Shutting down. Sending TERM signal to child processes ****" + # Kill the entire process group, which includes all background jobs + kill -TERM 0 + wait + echo "DONE" +} + +# Function to launch jobs for a specific machine +function launch_jobs() { + local machine=$1 + shift # The rest of $@ are the forwarded arguments + local UBERENV_HELPER="./scripts/setupLC-TPL-uberenv-helper.bash" + local COMMON="^vtk generator=ninja" + + echo "-----> Launching jobs for [${machine}]..." + + # Note: The max. time allowed on the debug queue is 1h. If we need more, switch to pbatch + case "$machine" in + dane) + ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-14 "+docs %clang-14 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-19 "+docs %clang-19 ${COMMON}" "${ALLOC_CMD}" "$@" & + ;; + + matrix) + ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-14-cuda-12.6 "+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-19-cuda-12.9 "+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & + ;; -echo "Building all LC TPLs from $GEOS_BRANCH to be installed at $INSTALL_DIR..." + tuo|tuolumne) + ALLOC_CMD="salloc -N 1 --exclusive -t 60 -A vortex -ppdebug" + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne cce-20-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ${COMMON}" "${ALLOC_CMD}" "$@" & + "${UBERENV_HELPER}" "$INSTALL_DIR" tuolumne llvm-amdgpu-6.4.2-rocm-6.4.2 "+rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942 %llvm-amdgpu_6_4_2 ${COMMON}" "${ALLOC_CMD}" "$@" & + ;; -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane gcc-12 "%gcc@12.1.1 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane gcc-12noAVX "%gcc@12noAVX +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane gcc-13 "%gcc@13.3.1 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane clang-14 "%clang@14.0.6 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR dane clang-19 "%clang@19.1.3 +docs" "salloc -N 1 -n 112 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix gcc-12-cuda-12.6 "%gcc@12.1.1+cuda~uncrustify cuda_arch=90 ^cuda@12.6.0+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix gcc-13-cuda-12.9 "%gcc@13.3.1+cuda~uncrustify cuda_arch=90 ^cuda@12.9.1+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix clang-14-cuda-12.6 "%clang@14.0.6+cuda~uncrustify cuda_arch=90 ^cuda@12.6.0+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR matrix clang-19-cuda-12.9 "%clang@19.1.3+cuda~uncrustify cuda_arch=90 ^cuda@12.9.1+allow-unsupported-compilers" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & -./scripts/setupLC-TPL-uberenv-helper.bash $INSTALL_DIR tuolumne cce-20-rocm-6.4.2 "%cce@20.0.0 +rocm~pygeosx~trilinos~petsc~docs amdgpu_target=gfx942" "salloc -N 1 --exclusive -t 120 -A vortex" $@ & + *) + echo "WARNING: Unknown machine '$machine'. Skipping." >&2 + ;; + esac +} + +# --- Main Execution --- +echo "Building TPLs for machines: ${MACHINES_TO_RUN[*]}" +echo "Installation directory: $INSTALL_DIR" +echo "Forwarded arguments: ${FORWARDED_ARGS[*]}" +if [ "$SET_PERMISSIONS" = false ]; then + echo "Permissions: SKIPPING all chmod/chgrp calls." +fi +echo "---" +for machine in "${MACHINES_TO_RUN[@]}"; do + launch_jobs "$machine" "${FORWARDED_ARGS[@]}" +done + +echo "All jobs launched. Waiting for completion..." # Note: Estimated completion time is ~90 minutes. # Check log files for unreported completion of jobs. wait -chmod -R g+rx $INSTALL_DIR -chgrp -R GEOS $INSTALL_DIR +# --- Conditionally Set Final Permissions --- +if [ "$SET_PERMISSIONS" = true ]; then + echo "---" + echo "Finalizing permissions..." + chmod -R g+rx "$INSTALL_DIR" + chgrp -R GEOS "$INSTALL_DIR" +else + echo "---" + echo "Skipping final permission updates as requested." +fi -echo "Complete" +echo "Complete." diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml index 71c542f7..7e898860 100644 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml @@ -16,13 +16,13 @@ # geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@10.0.1+cuda~uncrustify cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" # -# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@8.3.1+cuda~uncrustify cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" # -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@13.0.1+cuda~uncrustify cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" # -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@13.0.1+cuda~openmp~uncrustify cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers" +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" spack: @@ -44,79 +44,109 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - spec: clang@10.0.1 - paths: - cc: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] - - compiler: - spec: clang@13.0.1 - paths: - cc: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@8.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@12.2.1 - paths: - cc: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - operating_system: rhel7 - target: ppc64le - modules: [] - extra_rpaths: [] + toolchains: + clang-13: + - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@8.3.1' + when: '%fortran' + - spec: '%spectrum-mpi@release.clang.13' + when: '%mpi' + clang-10: + - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' + when: '%c' + - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%[virtuals=fortran]gcc@8.3.1' + when: '%fortran' + - spec: '%spectrum-mpi@release.clang.10' + when: '%mpi' + gcc-12: + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%spectrum-mpi@release.gcc.12' + when: '%mpi' + gcc-8: + - spec: '%c=gcc@8.3.1 ' + when: '%c' + - spec: '%cxx=gcc@8.3.1 ' + when: '%cxx' + - spec: '%fortran=gcc@8.3.1 ' + when: '%fortran' + - spec: '%spectrum-mpi@release.gcc.8' + when: '%mpi' packages: - all: - compiler: [gcc, clang] - providers: - mpi: [spectrum-mpi] - blas: [essl] - lapack: [essl] + mpi: + require: + - spectrum-mpi + blas: + require: + - essl + lapack: + require: + - essl + + llvm: + externals: + - spec: llvm@10.0.1+clang~flang~lld~lldb + prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang + cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ + flags: {} + extra_rpaths: [] + - spec: llvm@13.0.1+clang~flang~lld~lldb + prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang + cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ + flags: {} + extra_rpaths: [] + gcc: + externals: + - spec: gcc@8.3.1 languages:='c,c++,fortran' + prefix: /usr/tce/packages/gcc/gcc-8.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran + flags: + # Fix for "undefined reference to `_gfortran_transfer_integer_write'" + ldlibs: -lgfortran + extra_rpaths: [] + - spec: gcc@12.2.1 languages:='c,c++,fortran' + prefix: /usr/tce/packages/gcc/gcc-12.2.1 + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran + flags: + # Fix for "undefined reference to `_gfortran_transfer_integer_write'" + ldlibs: -lgfortran + extra_rpaths: [] spectrum-mpi: buildable: False externals: - - spec: spectrum-mpi@release%gcc@8.3.1 + - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release%gcc@12.2.1 + - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release%clang@10.0.1 + - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release%clang@13.0.1 + - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 essl: diff --git a/scripts/spack_configs/pangea-4/spack.yaml b/scripts/spack_configs/pangea-4/spack.yaml index 26d14514..aabf1f14 100644 --- a/scripts/spack_configs/pangea-4/spack.yaml +++ b/scripts/spack_configs/pangea-4/spack.yaml @@ -5,7 +5,7 @@ # # Run command from the top-level of the repository: # ./scripts/uberenv/uberenv.py \ -# --spec "%gcc@12.2.1 ~openmp~pygeosx+docs" \ +# --spec "~openmp~pygeosx+docs %gcc-12" \ # --spack-env-file=scripts/spack_configs/pangea-4/spack.yaml \ # --project-json=.uberenv_config.json \ # --prefix ${GEOS_TPL_DIR} @@ -17,8 +17,7 @@ spack: all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage + build_stage: $spack/../build_stage # Regular TPLs do not need views view: false @@ -28,47 +27,39 @@ spack: - ../defaults.yaml - ../versions.yaml - ############# - # COMPILERS # - ############# - - compilers: - - compiler: - spec: gcc@=12.2.1 - paths: - cc: /opt/rh/gcc-toolset-12/root/usr/bin/gcc - cxx: /opt/rh/gcc-toolset-12/root/usr/bin/g++ - f77: /opt/rh/gcc-toolset-12/root/usr/bin/gfortran - fc: /opt/rh/gcc-toolset-12/root/usr/bin/gfortran - flags: - cflags: -march=native -mtune=native - cxxflags: -march=native -mtune=native - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] + ################ + # TOOLCHAINS # + ################ + toolchains: + gcc-12: + - spec: '%c=gcc@12.2.1' + when: '%c' + - spec: '%cxx=gcc@12.2.1' + when: '%cxx' + - spec: '%fortran=gcc@12.2.1' + when: '%fortran' + - spec: '%hpcx-mpi@2.20.0' + when: '%mpi' ############# # PACKAGES # ############# - packages: all: target: [znver3] - compiler: [gcc] - providers: - blas: [intel-oneapi-mkl] - lapack: [intel-oneapi-mkl] - mpi: [hpcx-mpi] - #### - # make sure spack doesn't rebuild mpi, blas and lapack libs - # without this it will try to build mpich and openblas + # Ensure concretizer uses external MPI/BLAS/LAPACK implementation mpi: + require: + - hpcx-mpi buildable: false blas: + require: + - "intel-oneapi-mkl" buildable: false lapack: + require: + - "intel-oneapi-mkl" buildable: false #### @@ -77,12 +68,35 @@ spack: require: "@3.26.1" grpc: require: "@1.64.0-full-clone" - - # spec of spack packages to reuse + #### + # external compiler + gcc: + externals: + - spec: gcc@12.2.1 languages:=c,c++,fortran + prefix: /opt/rh/gcc-toolset-12/root/usr + extra_attributes: + compilers: + c: /opt/rh/gcc-toolset-12/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-12/root/usr/bin/g++ + fortran: /opt/rh/gcc-toolset-12/root/usr/bin/gfortran + flags: + cflags: -march=native -mtune=native + cxxflags: -march=native -mtune=native + operating_system: rhel8 + target: x86_64 + extra_rpaths: [] + + #### + # specs of spack packages to reuse + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /lustre/p4scratch/data_local/sw/RHEL8/spack-0.21.0/opt/gcc-12.2.1/bison-3.8.2-ozy3z5u intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2023.2.0 threads=openmp + - spec: intel-oneapi-mkl@2023.2.0%gcc@12.2.1 prefix: /lustre/p4scratch/data_local/sw/RHEL8/spack-0.21.0/opt/gcc-12.1/intel-oneapi-mkl-2023.2.0-7kkfgmz buildable: False python: @@ -125,11 +139,25 @@ spack: - spec: gmp@6.2.1 prefix: /lustre/p4scratch/data_local/sw/RHEL8/spack-0.21.0/opt/gcc-12.1/gmp-6.2.1-pwaztmh buildable: false + flex: + externals: + - spec: flex@2.6.3 + prefix: /lustre/p4scratch/data_local/sw/RHEL8/spack-0.21.0/opt/gcc-8.5.0/flex-2.6.3-xyx4nx4 + buildable: false + doxygen: + externals: + - spec: doxygen@1.8.20 + prefix: /lustre/p4scratch/data_local/sw/RHEL8/spack-0.21.0/opt/gcc-12.2.1/doxygen-1.8.20-idfxpwl #### # spec of system packages to reuse # bin + gettext: + externals: + - spec: gettext@0.19.8.1 + prefix: /usr + buildable: false git: externals: - spec: git@2.39.3~tcltk @@ -145,6 +173,11 @@ spack: - spec: m4@1.4.18 prefix: /usr buildable: false + xz: + externals: + - spec: xz@5.2.4 + prefix: /usr + buildable: false # libs gcc-runtime: externals: diff --git a/scripts/spack_configs/pine/spack.yaml b/scripts/spack_configs/pine/spack.yaml index bafae683..3941d5dc 100644 --- a/scripts/spack_configs/pine/spack.yaml +++ b/scripts/spack_configs/pine/spack.yaml @@ -1,14 +1,11 @@ -# This is a Spack Environment file for Pine. -# -# It describes a set of packages to be installed, along with -# configuration settings. -# +# This is a Spack Environment file for Pine (Spack 1.0.2) # Run command from the top-level of the repository: # ./scripts/uberenv/uberenv.py \ -# --spec "%gcc@11.4.1 ~openmp~pygeosx~docs" \ +# --spec "~openmp~pygeosx~docs" \ # --spack-env-file=scripts/spack_configs/pine/spack.yaml \ # --project-json=.uberenv_config.json \ # --prefix ${GEOS_TPL_DIR} + spack: config: install_tree: @@ -19,130 +16,170 @@ spack: test_stage: $spack/../test_stage build_stage: - $spack/../build_stage - # Regular TPLs do not need views + build_jobs: 16 + view: false - # Include shared variants and versions include: - ../defaults.yaml - ../versions.yaml - ############# - # COMPILERS # - ############# - compilers: - - compiler: - spec: gcc@11.4.1 - paths: - cc: /usr/bin/gcc - cxx: /usr/bin/g++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: - cflags: -march=native -mtune=native - cxxflags: -march=native -mtune=native - operating_system: rocky9 - target: x86_64 - modules: [] - extra_rpaths: [] - - ############# - # PACKAGES # - ############# + toolchains: + gcc-11: + - spec: '%c=gcc@11.4.1' + when: '%c' + - spec: '%cxx=gcc@11.4.1' + when: '%cxx' + - spec: '%fortran=gcc@11.4.1' + when: '%fortran' + - spec: '%openmpi@5.0.5' + when: '%mpi' + packages: all: target: [zen4] - compiler: [gcc] providers: - blas: [intel-mkl] - lapack: [intel-mkl] + blas: [openblas] + lapack: [openblas] mpi: [openmpi] + zlib-api: [zlib] + pkgconfig: [pkgconf] - #### - # make sure spack doesn't rebuild mpi, blas and lapack libs - mpi: + # Force Spack to build openblas, not use system version + openblas: + buildable: true + require: "@0.3.27 threads=none" + + # Disable unwanted compilers + nvhpc: + buildable: false + intel: buildable: false - blas: + oneapi: + buildable: false + cce: + buildable: false + llvm-amdgpu: + buildable: false + + gcc: + externals: + - spec: gcc@=11.4.1 + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + flags: + cflags: -march=native -mtune=native + cxxflags: -march=native -mtune=native + environment: {} + extra_rpaths: [] buildable: false - lapack: + + gcc-runtime: buildable: false + externals: + - spec: gcc-runtime@11.4.1 + prefix: /usr/lib/gcc/x86_64-redhat-linux/11 - #### - # spec of spack packages to reuse gmp: externals: - spec: gmp@6.3.0 prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/gmp-6.3.0-q4glkxa77fd4rejkrrsuydut2c222fkm buildable: false + mpfr: externals: - spec: mpfr@4.2.0 prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/mpfr-4.2.1-eaxjflobjpilpwfe4aalusexhcxhptrf - buildable: false + buildable: false + libiconv: externals: - spec: libiconv@1.17 prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/libiconv-1.17-crgtppb7hj54qwkl3hfc6ejyu34gsa4d buildable: false + perl: externals: - spec: perl@5.38.2 prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/perl-5.38.2-wxq42supt6zs3f63ajjtp6423q6teyot buildable: false + berkeley-db: externals: - spec: berkeley-db@18.1.40 prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/berkeley-db-18.1.40-nixwddyxlb523z3aly4b7bje4rfft3me buildable: false + cmake: externals: - spec: cmake@3.29.6 prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/cmake-3.29.6-i23jd7kq7lyqvrpsguhznfhoy4mso6up buildable: false - #### - # spec of system packages to reuse - # bin git: externals: - spec: git@2.39.3~tcltk prefix: /usr buildable: false + gmake: externals: - spec: gmake@4.3 prefix: /usr buildable: false + m4: externals: - spec: m4@1.4.19 prefix: /usr buildable: false + python: - buildable: False + buildable: false externals: - spec: python@3.9.18 prefix: /usr - #lib + + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /usr + + ninja: + buildable: false + externals: + - spec: ninja@1.10.2 + prefix: /usr + openmpi: externals: - spec: openmpi@5.0.5 prefix: /hrtc/apps/mpi/openmpi/x86_64/rocky9/5.0.5/gcc/11.4.1/nocuda/ buildable: false - intel-mkl: - externals: - - spec: intel-mkl@219.5.281 - prefix: /apps/intel/2019/u5/compilers_and_libraries_2019.5.281/linux/mkl/ - buildable: false + pkgconf: externals: - spec: pkgconf@3.0.0 prefix: /usr buildable: false + readline: externals: - spec: readline@8.1 prefix: /usr - buildable: false + buildable: false + zlib: + buildable: false + externals: + - spec: zlib@1.2.11 + prefix: /usr + concretizer: + unify: true + reuse: false diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index ebb477b9..fc593828 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -10,18 +10,24 @@ # See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. #------------------------------------------------------------------------------------------------------------ -# geosx@develop%clang@14.0.6 -# geosx@develop%gcc@12.1.1 -# geosx@develop%gcc@noAVX -# # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="%clang@14.0.6 +docs" # -# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@12.1.1 +docs" +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-12" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %gcc-13" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %clang-14" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+docs %clang-19" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %gcc-12 ^cuda@12.6.0+allow-unsupported-compilers" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %clang-14 ^cuda@12.6.0+allow-unsupported-compilers" +# +# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=90 %clang-19 ^cuda@12.9.1+allow-unsupported-compilers" # -# No AVX instructions -# python3 ./scripts/uberenv/uberenv.py --spec="%gcc@12noAVX +docs" - spack: config: @@ -31,8 +37,8 @@ spack: all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage + build_stage: $spack/../build_stage + build_jobs: 112 # Regular TPLs do not need views view: false @@ -42,115 +48,139 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers:: - - compiler: - spec: clang@14.0.6 - paths: - cc: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang - cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: clang@19.1.3 - paths: - cc: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang - cxx: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@12.1.1 - paths: - cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@12noAVX - paths: - cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran - flags: - cxxflags: -march=x86-64-v2 -mno-avx512f - operating_system: rhel8 - target: x86_64 - modules: [] - extra_rpaths: [] - - compiler: - spec: gcc@13.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - operating_system: rhel8 - target: x86_64 - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] - - compiler: - spec: gcc@13noAVX - paths: - cc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ - f77: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran - flags: - cxxflags: -march=x86-64-v2 -mno-avx512f - operating_system: rhel8 - target: x86_64 - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] + toolchains: + clang-19: + - spec: '%c=llvm@19.1.3' + when: '%c' + - spec: '%cxx=llvm@19.1.3' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.clang_19' + when: '%mpi' + clang-14: + - spec: '%c=llvm@14.0.6' + when: '%c' + - spec: '%cxx=llvm@14.0.6' + when: '%cxx' + - spec: '%fortran=gcc@10.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.clang_14' + when: '%mpi' + gcc-12: + - spec: '%c=gcc@12.1.1' + when: '%c' + - spec: '%cxx=gcc@12.1.1' + when: '%cxx' + - spec: '%fortran=gcc@12.1.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' + gcc-13: + - spec: '%c=gcc@13.3.1' + when: '%c' + - spec: '%cxx=gcc@13.3.1' + when: '%cxx' + - spec: '%fortran=gcc@13.3.1' + when: '%fortran' + - spec: '%mvapich2@2.3.7.gcc' + when: '%mpi' packages: all: - target: [ivybridge] - compiler: [gcc, clang] - providers: - mpi: [mvapich2] - blas: [intel-oneapi-mkl] - lapack: [intel-oneapi-mkl] - - # Lock down which MPI we are using + target: [sapphirerapids] + + mpi: + require: + - mvapich2 + + zlib-api: + require: + - zlib + + blas: + require: + - "openblas" + lapack: + require: + - "openblas" + + # Compilers + llvm: + externals: + - spec: llvm@14.0.6+clang~flang~lld~lldb + prefix: /usr/tce/packages/clang/clang-14.0.6-magic/ + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ + extra_rpaths: [] + - spec: llvm@19.1.3+clang~lld~lldb + prefix: /usr/tce/packages/clang/clang-19.1.3 + extra_attributes: + compilers: + c: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++ + flags: {} + gcc: + externals: + - spec: gcc@10.3.1 languages:='c,c++,fortran' + prefix: /usr/tce/packages/gcc/gcc-10.3.1 + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-10.3.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-10.3.1/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran + flags: {} + environment: {} + extra_rpaths: [] + - spec: gcc@12.1.1 languages:=c,c++,fortran + prefix: /usr/tce/packages/gcc/gcc-12.1.1-magic/ + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + extra_rpaths: [] + - spec: gcc@13.3.1 languages:=c,c++,fortran + prefix: /usr/tce/packages/gcc/gcc-13.3.1-magic/ + extra_attributes: + compilers: + c: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/g++ + fortran: /usr/tce/packages/gcc/gcc-13.3.1-magic/bin/gfortran + extra_rpaths: [] + + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.3.1 + prefix: /usr/tce/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13 + - spec: gcc-runtime@12.1.1 + prefix: /usr/tce/packages/gcc/gcc-12.1.1/lib/gcc/x86_64-redhat-linux/12 + - spec: gcc-runtime@11.2.1 + prefix: /usr/tce/packages/gcc/gcc-11.2.1/lib/gcc/x86_64-redhat-linux/11 + + # Lock down which MPI we are using mvapich2: buildable: False externals: - - spec: mvapich2@2.3%gcc@12.1.1 process_managers=slurm + - spec: mvapich2@2.3.7.gcc process_managers=slurm %gcc@12.1.1 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic - - spec: mvapich2@2.3%gcc@13.3.1 process_managers=slurm + - spec: mvapich2@2.3.7.gcc process_managers=slurm %gcc@13.3.1 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-13.3.1-magic - - spec: mvapich2@2.3%gcc@12noAVX process_managers=slurm - prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic - - spec: mvapich2@2.3%clang@14.0.6 process_managers=slurm + - spec: mvapich2@2.3.7.clang_14 process_managers=slurm %llvm@14.0.6 prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6-magic - - spec: mvapich2@2.3%clang@19.1.3 process_managers=slurm - prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3-magic + - spec: mvapich2@2.3.7.clang_19 process_managers=slurm %llvm@19.1.3 + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3 + + # Spack issue prevents compilation: https://github.com/spack/spack/issues/49994 + # intel-oneapi-mkl: + # externals: + # - spec: intel-oneapi-mkl@2022.1.0 threads=openmp %gcc@12.1.1 + # prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ + # buildable: False - intel-oneapi-mkl: - externals: - - spec: intel-oneapi-mkl@2022.1.0 threads=openmp - prefix: /usr/tce/packages/mkl/mkl-2022.1.0/ - buildable: False cuda: buildable: False externals: @@ -158,6 +188,7 @@ spack: prefix: /usr/tce/packages/cuda/cuda-12.6.0 - spec: cuda@12.9.1 +allow-unsupported-compilers prefix: /usr/tce/packages/cuda/cuda-12.9.1 + # System level packages to not build papi: buildable: False @@ -230,3 +261,29 @@ spack: externals: - spec: gettext@0.19.8.1 prefix: /usr/bin/ + openblas: + buildable: false + externals: + - prefix: /usr + spec: openblas@0.3.15 + zlib: + buildable: false + externals: + - spec: zlib@1.2.11 + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr/tce/packages/ninja/ninja-1.11.1 + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /collab/usr/gapps/python/build/spack-toss4.1/opt/spack/linux-rhel8-ivybridge/gcc-10.3.1/bison-3.8.2-ihsbaayrrhbmxufyobmimzq3lhxxdbck/ + # Avoid using external flex - issues compiling scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.1 + # prefix: /usr diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 814e4e5d..59f648a6 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -11,10 +11,12 @@ #------------------------------------------------------------------------------------------------------------ # geosx@develop%cce@20.0.0 +# geosx@develop%llvm-amdgpu_6_4_2 # # Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="%cce@20.0.0 +docs" +# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %cce-20 ^vtk generator=ninja" # +# python3 ./scripts/uberenv/uberenv.py --spec="+rocm~pygeosx+hypre~trilinos~petsc~docs amdgpu_target=gfx942 %llvm-amdgpu_6_4_2 ^vtk generator=ninja" spack: @@ -26,7 +28,7 @@ spack: misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage build_stage: $spack/../build_stage - build_jobs: 48 + build_jobs: 96 # Regular TPLs do not need views view: false @@ -36,76 +38,28 @@ spack: - ../defaults.yaml - ../versions.yaml - compilers: - - compiler: - spec: cce@=20.0.0 - paths: - cc: /usr/tce/packages/cce-tce/cce-20.0.0/bin/craycc - cxx: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayCC - f77: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn - fc: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - PrgEnv-cray/8.6.0 - - craype-x86-genoa - - craype/2.7.35 - - craype-network-ofi - - craype-accel-amd-gfx942 - - cray-mpich/9.0.1 - - cray-libsci/25.09.0 - - cce/20.0.0 - - libfabric/2.1 - - flux_wrappers/0.1 - - StdEnv - - perftools-base/25.09.0 - - xpmem/2.6.5 - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: - - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - - /opt/cray/pe/mpich/9.0.1/gtl/lib - - compiler: - spec: rocmcc@=6.4.2 - paths: - cc: /opt/rocm-6.4.2/llvm/bin/amdclang - cxx: /opt/rocm-6.4.2/llvm/bin/amdclang++ - f77: /opt/rocm-6.4.2/llvm/bin/amdflang - fc: /opt/rocm-6.4.2/llvm/bin/amdflang - flags: {} - operating_system: rhel8 - target: x86_64 - modules: - - rocm/6.4.2 - - PrgEnv-cray/8.6.0 - - craype-x86-genoa - - craype/2.7.35 - - craype-network-ofi - - craype-accel-amd-gfx942 - - cray-mpich/9.0.1 - - cray-libsci/25.09.0 - - cce/20.0.0 - - libfabric/2.1 - - flux_wrappers/0.1 - - StdEnv - - perftools-base/25.09.0 - - xpmem/2.6.5 - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: - - /opt/rocm-6.4.2/lib - - /opt/rocm-6.4.2/llvm/lib - - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - - /opt/cray/pe/mpich/9.0.1/gtl/lib + toolchains: + llvm-amdgpu_6_4_2: + - spec: '%c=llvm-amdgpu@6.4.2' + when: '%c' + - spec: '%cxx=llvm-amdgpu@6.4.2' + when: '%cxx' + - spec: '%fortran=llvm-amdgpu@6.4.2' + when: '%fortran' + - spec: '%cray-mpich@9.0.1_llvm_amdgpu' + when: '%mpi' + cce-20: + - spec: '%c=cce@20.0.0' + when: '%c' + - spec: '%cxx=cce@20.0.0' + when: '%cxx' + - spec: '%fortran=cce@20.0.0' + when: '%fortran' + - spec: '%cray-mpich@9.0.1_cce' + when: '%mpi' packages: all: - require: target=zen4 providers: blas: [netlib-lapack] lapack: [netlib-lapack] @@ -113,14 +67,94 @@ spack: zlib-api: [zlib] pkgconfig: [pkgconf] + cce: + externals: + - spec: cce@=20.0.0 + prefix: /usr/tce/packages/cce-tce/cce-20.0.0 + extra_attributes: + compilers: + c: /usr/tce/packages/cce-tce/cce-20.0.0/bin/craycc + cxx: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayCC + fortran: /usr/tce/packages/cce-tce/cce-20.0.0/bin/crayftn + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib + - /opt/cray/pe/mpich/9.0.1/gtl/lib + modules: + - PrgEnv-cray/8.6.0 + - craype-x86-genoa + - craype/2.7.35 + - craype-network-ofi + - craype-accel-amd-gfx942 + - cray-mpich/9.0.1 + - cray-libsci/25.09.0 + - cce/20.0.0 + - libfabric/2.1 + - flux_wrappers/0.1 + - StdEnv + - perftools-base/25.09.0 + - xpmem/2.6.5 + + + llvm-amdgpu: + externals: + - spec: llvm-amdgpu@=6.4.2 + prefix: /opt/rocm-6.4.2/llvm + extra_attributes: + compilers: + c: /opt/rocm-6.4.2/llvm/bin/amdclang + cxx: /opt/rocm-6.4.2/llvm/bin/amdclang++ + fortran: /opt/rocm-6.4.2/llvm/bin/amdflang + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/rocm-6.4.2/lib + - /opt/rocm-6.4.2/llvm/lib + - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib + - /opt/cray/pe/mpich/9.0.1/gtl/lib + modules: + - rocm/6.4.2 + - PrgEnv-cray/8.6.0 + - craype-x86-genoa + - craype/2.7.35 + - craype-network-ofi + - craype-accel-amd-gfx942 + - cray-mpich/9.0.1 + - cray-libsci/25.09.0 + - cce/20.0.0 + - libfabric/2.1 + - flux_wrappers/0.1 + - StdEnv + - perftools-base/25.09.0 + - xpmem/2.6.5 + + gcc-runtime: + buildable: False + externals: + - spec: gcc-runtime@13.3.1 + prefix: /usr/tce/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13 + - spec: gcc-runtime@12.1.1 + prefix: /usr/tce/packages/gcc/gcc-12.1.1/lib/gcc/x86_64-redhat-linux/12 + - spec: gcc-runtime@11.2.1 + prefix: /usr/tce/packages/gcc/gcc-11.2.1/lib/gcc/x86_64-redhat-linux/11 + # Lock down which MPI we are using cray-mpich: buildable: false externals: - - spec: cray-mpich@9.0.1%cce@20.0.0 - prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-cce-20.0.0-magic - - spec: cray-mpich@9.0.1%rocmcc@6.4.2 - prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-rocmcc-6.4.2-cce-20.0.0-magic + - spec: cray-mpich@9.0.1_cce %cce@20.0.0 + # prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-cce-20.0.0-magic + prefix: /usr/tce/packages/cray-mpich-tce/cray-mpich-9.0.1-rocmcc-6.4.2-cce-20.0.0/ + - spec: cray-mpich@9.0.1_llvm_amdgpu %llvm-amdgpu@6.4.2 + # prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-rocmcc-6.4.2-cce-20.0.0-magic + prefix: /usr/tce/packages/cray-mpich-tce/cray-mpich-9.0.1-rocmcc-6.4.2/ # ROCm packages hip: @@ -163,14 +197,6 @@ spack: prefix: /opt/rocm-6.4.0 - spec: hipsolver@6.4.2 prefix: /opt/rocm-6.4.2 - llvm-amdgpu: - version: [6.4.0, 6.4.2] - buildable: false - externals: - - spec: llvm-amdgpu@6.4.0 - prefix: /opt/rocm-6.4.0/llvm - - spec: llvm-amdgpu@6.4.2 - prefix: /opt/rocm-6.4.2/llvm hsa-rocr-dev: version: [6.4.0, 6.4.2] buildable: false @@ -421,4 +447,20 @@ spack: buildable: false externals: - spec: zlib@1.2.11 - prefix: /usr \ No newline at end of file + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.10.2 + prefix: /usr/tce/packages/ninja/ninja-1.10.2 + # Avoid using external flex - issues compiling scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.1 + # prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /collab/usr/gapps/python/build/spack-toss4.1/opt/spack/linux-rhel8-ivybridge/gcc-10.3.1/bison-3.8.2-ihsbaayrrhbmxufyobmimzq3lhxxdbck/ diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py new file mode 100644 index 00000000..16b68a56 --- /dev/null +++ b/scripts/spack_packages/packages/chai/package.py @@ -0,0 +1,9 @@ +import os + +from spack.package import * +from spack_repo.builtin.packages.chai.package import Chai as BuiltinChai + +class Chai(BuiltinChai): + # Bypass llnl_link_helpers failure + depends_on("fortran") + diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py index f9d0564d..fa2a2cb7 100644 --- a/scripts/spack_packages/packages/essl/package.py +++ b/scripts/spack_packages/packages/essl/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack.package import * - +from spack_repo.builtin.build_systems.bundle import BundlePackage # Recipe for pre-built essl library on blueos machines. # Defines additonal flags for blueos: diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 6405d322..2fd7fbe0 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * import warnings import socket @@ -14,6 +14,11 @@ from os import environ as env from os.path import join as pjoin +from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_path + +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.cuda import CudaPackage + # Tested specs are located at scripts/spack_configs/<$SYS_TYPE>/spack.yaml (e.g. %clang@10.0.1) # WARNING: +petsc variant is yet to be tested. @@ -85,10 +90,13 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): description='Add support for addr2line.') variant('mathpresso', default=True, description='Build mathpresso.') - variant('cuda_stack_size', default=0, description="Defines the adjusted cuda stack \ + variant('cuda_stack_size', default="0", description="Defines the adjusted cuda stack \ size limit if required. Zero or negative keep default behavior") # SPHINX_BEGIN_DEPENDS + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build") depends_on('cmake@3.24:', type='build') @@ -178,8 +186,8 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): with when("+hypre"): depends_on("hypre +superlu-dist+mixedint+mpi~shared cflags='-fPIC' cxxflags='-fPIC'", when='~cuda~rocm') - depends_on("hypre +cuda+superlu-dist+mixedint+mpi+umpire~shared cflags='-fPIC' cxxflags='-fPIC'", when='+cuda') - depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire~shared cflags='-fPIC' cxxflags='-fPIC'", when='+rocm') + depends_on("hypre +cuda+superlu-dist+mixedint+mpi+umpire+unified-memory~shared cflags='-fPIC' cxxflags='-fPIC'", when='+cuda') + depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire+unified-memory~shared cflags='-fPIC' cxxflags='-fPIC'", when='+rocm') depends_on("hypre ~openmp", when="~openmp") depends_on("hypre +openmp", when="+openmp") @@ -247,6 +255,7 @@ def _get_sys_type(self, spec): def _get_host_config_path(self, spec, lvarray=False): var = '' + if '+cuda' in spec: var = '-'.join([var, 'cuda']) var += "@" + str(spec['cuda'].version) @@ -260,7 +269,7 @@ def _get_host_config_path(self, spec, lvarray=False): if lvarray: hostname = "lvarray-" + hostname - host_config_path = "%s-%s-%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler)).replace('=',''), var) + host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler.name)), str(spec.compiler.version),var) dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) @@ -662,9 +671,9 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): # Lassen if sys_type in ('blueos_3_ppc64le_ib_p9'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) - # Ruby + # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm56')) + cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): """ diff --git a/scripts/spack_packages/packages/hdf5/package.py b/scripts/spack_packages/packages/hdf5/package.py deleted file mode 100644 index 061251f5..00000000 --- a/scripts/spack_packages/packages/hdf5/package.py +++ /dev/null @@ -1,747 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# GEOS_EDIT_START -# Older hdf5 spack recipe that works on lassen -# (latest spack release fails with MPI related configuration failure) -# GEOS_EDIT_END - -import os -import shutil - -import llnl.util.tty as tty - -from spack.package import * - - -class Hdf5(CMakePackage): - """HDF5 is a data model, library, and file format for storing and managing - data. It supports an unlimited variety of datatypes, and is designed for - flexible and efficient I/O and for high volume and complex data. - """ - - homepage = "https://portal.hdfgroup.org" - url = "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.8/src/hdf5-1.10.8.tar.gz" - list_url = "https://support.hdfgroup.org/ftp/HDF5/releases" - list_depth = 3 - git = "https://github.com/HDFGroup/hdf5.git" - maintainers = [ - "lrknox", - "brtnfld", - "byrnHDF", - "ChristopherHogan", - "epourmal", - "gheber", - "hyoklee", - "lkurz", - "soumagne", - ] - - tags = ["e4s"] - - test_requires_compiler = True - - # The 'develop' version is renamed so that we could uninstall (or patch) it - # without affecting other develop version. - version("develop-1.13", branch="develop") - version("develop-1.12", branch="hdf5_1_12") - version("develop-1.10", branch="hdf5_1_10") - version("develop-1.8", branch="hdf5_1_8") - - # Odd versions are considered experimental releases - version("1.13.2", sha256="01643fa5b37dba7be7c4db6bbf3c5d07adf5c1fa17dbfaaa632a279b1b2f06da") - - # Even versions are maintenance versions - version( - "1.12.2", - sha256="2a89af03d56ce7502dcae18232c241281ad1773561ec00c0f0e8ee2463910f14", - preferred=True, - ) - version( - "1.12.1", - sha256="79c66ff67e666665369396e9c90b32e238e501f345afd2234186bfb8331081ca", - preferred=True, - ) - version( - "1.12.0", - sha256="a62dcb276658cb78e6795dd29bf926ed7a9bc4edf6e77025cd2c689a8f97c17a", - preferred=True, - ) - version( - "1.10.9", - sha256="f5b77f59b705a755a5a223372d0222c7bc408fe8db6fa8d9d7ecf8bce291b8dd", - preferred=True, - ) - version( - "1.10.8", - sha256="d341b80d380dd763753a0ebe22915e11e87aac4e44a084a850646ff934d19c80", - preferred=True, - ) - version( - "1.10.7", - sha256="7a1a0a54371275ce2dfc5cd093775bb025c365846512961e7e5ceaecb437ef15", - preferred=True, - ) - version( - "1.10.6", - sha256="5f9a3ee85db4ea1d3b1fa9159352aebc2af72732fc2f58c96a3f0768dba0e9aa", - preferred=True, - ) - version( - "1.10.5", - sha256="6d4ce8bf902a97b050f6f491f4268634e252a63dadd6656a1a9be5b7b7726fa8", - preferred=True, - ) - version( - "1.10.4", - sha256="8f60dc4dd6ab5fcd23c750d1dc5bca3d0453bdce5c8cdaf0a4a61a9d1122adb2", - preferred=True, - ) - version( - "1.10.3", - sha256="b600d7c914cfa80ae127cd1a1539981213fee9994ac22ebec9e3845e951d9b39", - preferred=True, - ) - version( - "1.10.2", - sha256="bfec1be8c366965a99812cf02ddc97e4b708c1754fccba5414d4adccdc073866", - preferred=True, - ) - version( - "1.10.1", - sha256="048a9d149fb99aaa1680a712963f5a78e9c43b588d0e79d55e06760ec377c172", - preferred=True, - ) - version( - "1.10.0-patch1", - sha256="6e78cfe32a10e6e0629393cdfddf6cfa536571efdaf85f08e35326e1b4e9eff0", - preferred=True, - ) - version( - "1.10.0", - sha256="81f6201aba5c30dced5dcd62f5d5477a2790fd5850e02ac514ca8bf3e2bb375a", - preferred=True, - ) - version( - "1.8.22", - sha256="8406d96d9355ef8961d2739fb8fd5474ad4cdf52f3cfac657733defd9709bfaa", - preferred=True, - ) - version( - "1.8.21", - sha256="87d8c82eba5cf766d97cd06c054f4639c1049c4adeaa3a79f77f8bd374f80f37", - preferred=True, - ) - version( - "1.8.19", - sha256="a4335849f19fae88c264fd0df046bc321a78c536b2548fc508627a790564dc38", - preferred=True, - ) - version( - "1.8.18", - sha256="cdb195ad8d9e6782acf24b2488061289f615628c2ccda8457b0a0c3fb7a8a063", - preferred=True, - ) - version( - "1.8.17", - sha256="d9cda297ee76ade9881c4208987939250d397bae6252d0ccb66fa7d24d67e263", - preferred=True, - ) - version( - "1.8.16", - sha256="ed17178abd9928a7237f30370189ba767b9e39e0db45917c2ac4665eb9cb4771", - preferred=True, - ) - version( - "1.8.15", - sha256="4e963216b7d32469596bc1321a8c3f6e0c278dcbbdb7be6414c63c081b34c275", - preferred=True, - ) - version( - "1.8.14", - sha256="1dbefeeef7f591897c632b2b090db96bb8d35ad035beaa36bc39cb2bc67e0639", - preferred=True, - ) - version( - "1.8.13", - sha256="82f6b38eec103b4fccfbf14892786e0c27a8135d3252d8601cf5bf20066d38c1", - preferred=True, - ) - version( - "1.8.12", - sha256="b5cccea850096962b5fd9e96f22c4f47d2379224bb41130d9bc038bb6c37dfcb", - preferred=True, - ) - version( - "1.8.10", - sha256="4813b79c5fb8701a625b9924b8203bc7154a77f9b826ad4e034144b4056a160a", - preferred=True, - ) - - variant("shared", default=True, description="Builds a shared version of the library") - - variant("hl", default=False, description="Enable the high-level library") - variant("cxx", default=False, description="Enable C++ support") - variant("fortran", default=False, description="Enable Fortran support") - variant("java", when="@1.10:", default=False, description="Enable Java support") - variant("threadsafe", default=False, description="Enable thread-safe capabilities") - variant("tools", default=True, description="Enable building tools") - variant("mpi", default=True, description="Enable MPI support") - variant("szip", default=False, description="Enable szip support") - # Build HDF5 with API compatibility. - variant( - "api", - default="default", - description="Choose api compatibility for earlier version", - values=("default", "v114", "v112", "v110", "v18", "v16"), - multi=False, - ) - - depends_on("cmake@3.12:", type="build") - - depends_on("mpi", when="+mpi") - depends_on("java", type=("build", "run"), when="+java") - depends_on("szip", when="+szip") - depends_on("zlib@1.1.2:") - - # The compiler wrappers (h5cc, h5fc, etc.) run 'pkg-config'. - depends_on("pkgconfig", type="run") - - conflicts("api=v114", when="@1.6:1.12", msg="v114 is not compatible with this release") - conflicts("api=v112", when="@1.6:1.10", msg="v112 is not compatible with this release") - conflicts("api=v110", when="@1.6:1.8", msg="v110 is not compatible with this release") - conflicts("api=v18", when="@1.6.0:1.6", msg="v18 is not compatible with this release") - - # The Java wrappers cannot be built without shared libs. - conflicts("+java", when="~shared") - # Fortran fails built with shared for old HDF5 versions - conflicts("+fortran", when="+shared@:1.8.15") - # See https://github.com/spack/spack/issues/31085 - conflicts("+fortran+mpi", when="@1.8.22") - - # GEOS_EDIT_START - # Commenting out unused patches - - # There are several officially unsupported combinations of the features: - # 1. Thread safety is not guaranteed via high-level C-API but in some cases - # it works. - # conflicts('+threadsafe+hl') - - # 2. Thread safety is not guaranteed via Fortran (CXX) API, but it's - # possible for a dependency tree to contain a package that uses Fortran - # (CXX) API in a single thread and another one that uses low-level C-API - # in multiple threads. To allow for such scenarios, we don't specify the - # following conflicts. - # conflicts('+threadsafe+cxx') - # conflicts('+threadsafe+fortran') - - # 3. Parallel features are not supported via CXX API, but for the reasons - # described in #2 we allow for such combination. - # conflicts('+mpi+cxx') - - # There are known build failures with intel@18.0.1. This issue is - # discussed and patch is provided at - # https://software.intel.com/en-us/forums/intel-fortran-compiler-for-linux-and-mac-os-x/topic/747951. - #patch("h5f90global-mult-obj-same-equivalence-same-common-block.patch", when="@1.10.1%intel@18") - - # Turn line comments into block comments to conform with pre-C99 language - # standards. Versions of hdf5 after 1.8.10 don't require this patch, - # either because they conform to pre-C99 or neglect to ask for pre-C99 - # language standards from their compiler. The hdf5 build system adds - # the -ansi cflag (run 'man gcc' for info on -ansi) for some versions - # of some compilers (see hdf5-1.8.10/config/gnu-flags). The hdf5 build - # system does not provide an option to disable -ansi, but since the - # pre-C99 code is restricted to just five lines of line comments in - # three src files, this patch accomplishes the simple task of patching the - # three src files and leaves the hdf5 build system alone. - #patch("pre-c99-comments.patch", when="@1.8.10") - - # There are build errors with GCC 8, see - # https://forum.hdfgroup.org/t/1-10-2-h5detect-compile-error-gcc-8-1-0-on-centos-7-2-solved/4441 - # patch( - # "https://salsa.debian.org/debian-gis-team/hdf5/raw/bf94804af5f80f662cad80a5527535b3c6537df6/debian/patches/gcc-8.patch", - # sha256="57cee5ff1992b4098eda079815c36fc2da9b10e00a9056df054f2384c4fc7523", - # when="@1.10.2%gcc@8:", - # ) - - # Disable MPI C++ interface when C++ is disabled, otherwise downstream - # libraries fail to link; see https://github.com/spack/spack/issues/12586 - # patch( - # "h5public-skip-mpicxx.patch", - # when="@1.8.10:1.8.21,1.10.0:1.10.5+mpi~cxx", - # sha256="b61e2f058964ad85be6ee5ecea10080bf79e73f83ff88d1fa4b602d00209da9c", - # ) - - # Fixes BOZ literal constant error when compiled with GCC 10. - # The issue is described here: https://github.com/spack/spack/issues/18625 - # patch( - # "hdf5_1.8_gcc10.patch", - # when="@:1.8.21", - # sha256="0e20187cda3980a4fdff410da92358b63de7ebef2df1d7a425371af78e50f666", - # ) - - # patch("fortran-kinds.patch", when="@1.10.7") - - # This patch may only be needed with GCC11.2 on macOS, but it's valid for - # any of the head HDF5 versions as of 12/2021. Since it's impossible to - # tell what Fortran version is part of a mixed apple-clang toolchain on - # macOS (which is the norm), and this might be an issue for other compilers - # as well, we just apply it to all platforms. - # See https://github.com/HDFGroup/hdf5/issues/1157 - # patch("fortran-kinds-2.patch", when="@1.10.8,1.12.1") - - # The argument 'buf_size' of the C function 'h5fget_file_image_c' is - # declared as intent(in) though it is modified by the invocation. As a - # result, aggressive compilers such as Fujitsu's may do a wrong - # optimization to cause an error. - # GEOS_EDIT_END - - def patch(self): - filter_file( - "INTEGER(SIZE_T), INTENT(IN) :: buf_size", - "INTEGER(SIZE_T), INTENT(OUT) :: buf_size", - "fortran/src/H5Fff.F90", - string=True, - ignore_absent=True, - ) - filter_file( - "INTEGER(SIZE_T), INTENT(IN) :: buf_size", - "INTEGER(SIZE_T), INTENT(OUT) :: buf_size", - "fortran/src/H5Fff_F03.f90", - string=True, - ignore_absent=True, - ) - if self.run_tests: - # hdf5 has ~2200 CPU-intensive tests, some of them have races: - # Often, these loop endless(at least on one Xeon and one EPYC). - # testphdf5 fails indeterministic. This fixes finishing the tests - filter_file( - "REMOVE_ITEM H5P_TESTS", - "REMOVE_ITEM H5P_TESTS t_bigio t_shapesame testphdf5", - "testpar/CMakeTests.cmake", - ) - - # The parallel compiler wrappers (i.e. h5pcc, h5pfc, etc.) reference MPI - # compiler wrappers and do not need to be changed. - filter_compiler_wrappers( - "h5cc", "h5hlcc", "h5fc", "h5hlfc", "h5c++", "h5hlc++", relative_root="bin" - ) - - def url_for_version(self, version): - url = ( - "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-{0}/hdf5-{1}/src/hdf5-{1}.tar.gz" - ) - return url.format(version.up_to(2), version) - - def flag_handler(self, name, flags): - spec = self.spec - cmake_flags = [] - - if name == "cflags": - if spec.compiler.name in ["gcc", "clang", "apple-clang", "oneapi"]: - # Quiet warnings/errors about implicit declaration of functions - # in C99: - cmake_flags.append("-Wno-error=implicit-function-declaration") - # Note that this flag will cause an error if building %nvhpc. - if spec.satisfies("@:1.8.12~shared"): - # More recent versions set CMAKE_POSITION_INDEPENDENT_CODE to - # True and build with PIC flags. - cmake_flags.append(self.compiler.cc_pic_flag) - elif name == "cxxflags": - if spec.satisfies("@:1.8.12+cxx~shared"): - cmake_flags.append(self.compiler.cxx_pic_flag) - elif name == "fflags": - if spec.satisfies("%cce+fortran"): - # Cray compiler generates module files with uppercase names by - # default, which is not handled by the CMake scripts. The - # following flag forces the compiler to produce module files - # with lowercase names. - cmake_flags.append("-ef") - if spec.satisfies("@:1.8.12+fortran~shared"): - cmake_flags.append(self.compiler.fc_pic_flag) - elif name == "ldlibs": - if "+fortran %fj" in spec: - cmake_flags.extend(["-lfj90i", "-lfj90f", "-lfjsrcinfo", "-lelf"]) - - return flags, None, (cmake_flags or None) - - @property - def libs(self): - """HDF5 can be queried for the following parameters: - - - "hl": high-level interface - - "cxx": C++ APIs - - "fortran": Fortran APIs - - "java": Java APIs - - :return: list of matching libraries - """ - query_parameters = self.spec.last_query.extra_parameters - - shared = "+shared" in self.spec - - # This map contains a translation from query_parameters - # to the libraries needed - query2libraries = { - tuple(): ["libhdf5"], - ("cxx", "fortran", "hl", "java"): [ - # When installed with Autotools, the basename of the real - # library file implementing the High-level Fortran interface is - # 'libhdf5hl_fortran'. Starting versions 1.8.22, 1.10.5 and - # 1.12.0, the Autotools installation also produces a symbolic - # link 'libhdf5_hl_fortran.' to - # 'libhdf5hl_fortran.'. Note that in the case of the - # dynamic library, the latter is a symlink to the real sonamed - # file 'libhdf5_fortran.so.'. This means that all - # dynamically linked executables/libraries of the dependent - # packages need 'libhdf5_fortran.so.' with the same - # DT_SONAME entry. However, the CMake installation (at least - # starting version 1.8.10) does not produce it. Instead, the - # basename of the library file is 'libhdf5_hl_fortran'. Which - # means that switching to CMake requires rebuilding of all - # dependant packages that use the High-level Fortran interface. - # Therefore, we do not try to preserve backward compatibility - # with Autotools installations by creating symlinks. The only - # packages that could benefit from it would be those that - # hardcode the library name in their building systems. Such - # packages should simply be patched. - "libhdf5_hl_fortran", - "libhdf5_hl_f90cstub", - "libhdf5_hl_cpp", - "libhdf5_hl", - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5_java", - "libhdf5", - ], - ("cxx", "hl"): [ - "libhdf5_hl_cpp", - "libhdf5_hl", - "libhdf5", - ], - ("fortran", "hl"): [ - "libhdf5_hl_fortran", - "libhdf5_hl_f90cstub", - "libhdf5_hl", - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5", - ], - ("hl",): [ - "libhdf5_hl", - "libhdf5", - ], - ("cxx", "fortran"): [ - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5_cpp", - "libhdf5", - ], - ("cxx",): [ - "libhdf5_cpp", - "libhdf5", - ], - ("fortran",): [ - "libhdf5_fortran", - "libhdf5_f90cstub", - "libhdf5", - ], - ("java",): [ - "libhdf5_java", - "libhdf5", - ], - } - - # Turn the query into the appropriate key - key = tuple(sorted(query_parameters)) - libraries = query2libraries[key] - - return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) - - @when("@:1.8.21,1.10.0:1.10.5+szip") - def setup_build_environment(self, env): - env.set("SZIP_INSTALL", self.spec["szip"].prefix) - - @run_before("cmake") - def fortran_check(self): - if "+fortran" in self.spec and not self.compiler.fc: - msg = "cannot build a Fortran variant without a Fortran compiler" - raise RuntimeError(msg) - - def cmake_args(self): - spec = self.spec - - if spec.satisfies("@:1.8.15+shared"): - tty.warn("hdf5@:1.8.15+shared does not produce static libraries") - - args = [ - # Always enable this option. This does not actually enable any - # features: it only *allows* the user to specify certain - # combinations of other arguments. - self.define("ALLOW_UNSUPPORTED", True), - # Speed-up the building by skipping the examples: - self.define("HDF5_BUILD_EXAMPLES", False), - self.define( - "BUILD_TESTING", - self.run_tests or - # Version 1.8.22 fails to build the tools when shared libraries - # are enabled but the tests are disabled. - spec.satisfies("@1.8.22+shared+tools"), - ), - self.define("HDF5_ENABLE_Z_LIB_SUPPORT", True), - self.define_from_variant("HDF5_ENABLE_SZIP_SUPPORT", "szip"), - self.define_from_variant("HDF5_ENABLE_SZIP_ENCODING", "szip"), - self.define_from_variant("BUILD_SHARED_LIBS", "shared"), - self.define("ONLY_SHARED_LIBS", False), - self.define_from_variant("HDF5_ENABLE_PARALLEL", "mpi"), - self.define_from_variant("HDF5_ENABLE_THREADSAFE", "threadsafe"), - self.define_from_variant("HDF5_BUILD_HL_LIB", "hl"), - self.define_from_variant("HDF5_BUILD_CPP_LIB", "cxx"), - self.define_from_variant("HDF5_BUILD_FORTRAN", "fortran"), - self.define_from_variant("HDF5_BUILD_JAVA", "java"), - self.define_from_variant("HDF5_BUILD_TOOLS", "tools"), - ] - - api = spec.variants["api"].value - if api != "default": - args.append(self.define("DEFAULT_API_VERSION", api)) - - if "+mpi" in spec: - args.append(self.define("CMAKE_C_COMPILER", spec["mpi"].mpicc)) - - if "+cxx" in self.spec: - args.append(self.define("CMAKE_CXX_COMPILER", spec["mpi"].mpicxx)) - - if "+fortran" in self.spec: - args.append(self.define("CMAKE_Fortran_COMPILER", spec["mpi"].mpifc)) - - # work-around for https://github.com/HDFGroup/hdf5/issues/1320 - if spec.satisfies("@1.10.8,1.13.0"): - args.append(self.define("HDF5_INSTALL_CMAKE_DIR", "share/cmake/hdf5")) - - return args - - @run_after("install") - def ensure_parallel_compiler_wrappers(self): - # When installed with Autotools and starting at least version 1.8.10, - # the package produces C compiler wrapper called either 'h5cc' (when MPI - # support is disabled) or 'h5pcc' (when MPI support is enabled). The - # CMake installation produces the wrapper called 'h5cc' (regardless of - # whether MPI support is enabled) only starting versions 1.8.21, 1.10.2 - # and 1.12.0. The current develop versions also produce 'h5pcc' when MPI - # support is enabled and the file is identical to 'h5cc'. Here, we make - # sure that 'h5pcc' is available when MPI support is enabled (only for - # versions that generate 'h5cc'). - if self.spec.satisfies("@1.8.21:1.8.22,1.10.2:1.10.7,1.12.0+mpi"): - with working_dir(self.prefix.bin): - # No try/except here, fix the condition above instead: - symlink("h5cc", "h5pcc") - - # The same as for 'h5pcc'. However, the CMake installation produces the - # Fortran compiler wrapper called 'h5fc' only starting versions 1.8.22, - # 1.10.6 and 1.12.0. The current develop versions do not produce 'h5pfc' - # at all. Here, we make sure that 'h5pfc' is available when Fortran and - # MPI support are enabled (only for versions that generate 'h5fc'). - if self.spec.satisfies( - "@1.8.22:1.8," "1.10.6:1.10," "1.12.0:1.12," "develop:" "+fortran+mpi" - ): - with working_dir(self.prefix.bin): - # No try/except here, fix the condition above instead: - symlink("h5fc", "h5pfc") - - @run_after("install") - def fix_package_config(self): - # We need to fix the pkg-config files, which are also used by the - # compiler wrappers. The files are created starting versions 1.8.21, - # 1.10.2 and 1.12.0. However, they are broken (except for the version - # 1.8.22): the files are named -.pc but reference - # packages. This was fixed in the develop versions at some point: the - # files started referencing - packages but got broken - # again: the files got names .pc but references had not been - # updated accordingly. Another issue, which we address here, is that - # some Linux distributions install pkg-config files named hdf5.pc and we - # want to override them. Therefore, the following solution makes sure - # that each -.pc file is symlinked by .pc and all - # references to - packages in the original files are - # replaced with references to packages. - pc_files = find(self.prefix.lib.pkgconfig, "hdf5*.pc", recursive=False) - - if not pc_files: - # This also tells us that the pkgconfig directory does not exist. - return - - # Replace versioned references in all pkg-config files: - filter_file( - r"(Requires(?:\.private)?:.*)(hdf5[^\s,]*)(?:-[^\s,]*)(.*)", - r"\1\2\3", - *pc_files, - backup=False - ) - - # Create non-versioned symlinks to the versioned pkg-config files: - with working_dir(self.prefix.lib.pkgconfig): - for f in pc_files: - src_filename = os.path.basename(f) - version_sep_idx = src_filename.find("-") - if version_sep_idx > -1: - tgt_filename = src_filename[:version_sep_idx] + ".pc" - if not os.path.exists(tgt_filename): - symlink(src_filename, tgt_filename) - - @run_after("install") - @on_package_attributes(run_tests=True) - def check_install(self): - self._check_install() - - def _check_install(self): - # Build and run a small program to test the installed HDF5 library - spec = self.spec - print("Checking HDF5 installation...") - checkdir = "spack-check" - with working_dir(checkdir, create=True): - # Because the release number in a develop branch is not fixed, - # only the major and minor version numbers are compared. - # Otherwise all 3 numbers are checked. - if "develop" in str(spec.version.up_to(3)): - source = r""" -#include -#include -#include -int main(int argc, char **argv) { - unsigned majnum, minnum, relnum; - herr_t herr = H5get_libversion(&majnum, &minnum, &relnum); - assert(!herr); - printf("HDF5 version %d.%d %u.%u\n", H5_VERS_MAJOR, H5_VERS_MINOR, - majnum, minnum); - return 0; -} -""" - else: - source = r""" -#include -#include -#include -int main(int argc, char **argv) { - unsigned majnum, minnum, relnum; - herr_t herr = H5get_libversion(&majnum, &minnum, &relnum); - assert(!herr); - printf("HDF5 version %d.%d.%d %u.%u.%u\n", H5_VERS_MAJOR, H5_VERS_MINOR, - H5_VERS_RELEASE, majnum, minnum, relnum); - return 0; -} -""" - expected = """\ -HDF5 version {version} {version} -""".format( - version=str(spec.version.up_to(3)) - ) - if "develop" in expected: - # Remove 'develop-' from the version in spack for checking - # version against the version in the HDF5 code. - expected = """\ -HDF5 version {version} {version} -""".format( - version=str(spec.version.up_to(3)).partition("-")[2] - ) - with open("check.c", "w") as f: - f.write(source) - if "+mpi" in spec: - cc = Executable(spec["mpi"].mpicc) - else: - cc = Executable(self.compiler.cc) - cc(*(["-c", "check.c"] + spec["hdf5"].headers.cpp_flags.split())) - cc(*(["-o", "check", "check.o"] + spec["hdf5"].libs.ld_flags.split())) - try: - check = Executable("./check") - output = check(output=str) - except ProcessError: - output = "" - success = output == expected - if not success: - print("Produced output does not match expected output.") - print("Expected output:") - print("-" * 80) - print(expected) - print("-" * 80) - print("Produced output:") - print("-" * 80) - print(output) - print("-" * 80) - raise RuntimeError("HDF5 install check failed") - shutil.rmtree(checkdir) - - def _test_check_versions(self): - """Perform version checks on selected installed package binaries.""" - spec_vers_str = "Version {0}".format(self.spec.version) - if "develop" in spec_vers_str: - # Remove 'develop-' from the version in spack for checking - # version against the version in the HDF5 code. - spec_vers_str = spec_vers_str.partition("-")[2] - - exes = [ - "h5copy", - "h5diff", - "h5dump", - "h5format_convert", - "h5ls", - "h5mkgrp", - "h5repack", - "h5stat", - "h5unjam", - ] - use_short_opt = ["h52gif", "h5repart", "h5unjam"] - for exe in exes: - reason = "test: ensuring version of {0} is {1}".format(exe, spec_vers_str) - option = "-V" if exe in use_short_opt else "--version" - self.run_test( - exe, option, spec_vers_str, installed=True, purpose=reason, skip_missing=True - ) - - def _test_example(self): - """This test performs copy, dump, and diff on an example hdf5 file.""" - test_data_dir = self.test_suite.current_test_data_dir - - filename = "spack.h5" - h5_file = test_data_dir.join(filename) - - reason = "test: ensuring h5dump produces expected output" - expected = get_escaped_text_output(test_data_dir.join("dump.out")) - self.run_test( - "h5dump", - filename, - expected, - installed=True, - purpose=reason, - skip_missing=True, - work_dir=test_data_dir, - ) - - reason = "test: ensuring h5copy runs" - options = ["-i", h5_file, "-s", "Spack", "-o", "test.h5", "-d", "Spack"] - self.run_test( - "h5copy", options, [], installed=True, purpose=reason, skip_missing=True, work_dir="." - ) - - reason = "test: ensuring h5diff shows no differences between orig and" " copy" - self.run_test( - "h5diff", - [h5_file, "test.h5"], - [], - installed=True, - purpose=reason, - skip_missing=True, - work_dir=".", - ) - - def test(self): - """Perform smoke tests on the installed package.""" - # Simple version check tests on known binaries - self._test_check_versions() - - # Run sequence of commands on an hdf5 file - self._test_example() - - # Run existing install check - self._check_install() \ No newline at end of file diff --git a/scripts/spack_packages/packages/llvm-amdgpu/package.py b/scripts/spack_packages/packages/llvm-amdgpu/package.py new file mode 100644 index 00000000..93dd170d --- /dev/null +++ b/scripts/spack_packages/packages/llvm-amdgpu/package.py @@ -0,0 +1,21 @@ +# Copyright 2013-2025 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +from spack.package import * +from spack_repo.builtin.packages.llvm_amdgpu.package import LlvmAmdgpu as BuiltinLlvmAmdgpu + +class LlvmAmdgpu(BuiltinLlvmAmdgpu): + + # PR that adds this change is pending: https://github.com/spack/spack-packages/pull/1557 + provides("fortran") + + # Fix from slack: + # https://spackpm.slack.com/archives/C08Q62S7XEX/p1751072888930439?thread_ts=1750704656.170759&cid=C08Q62S7XEX + compiler_wrapper_link_paths = { + "c": "rocmcc/amdclang", + "cxx": "rocmcc/amdclang++", + "fortran": "rocmcc/amdflang" + } diff --git a/scripts/spack_packages/packages/mathpresso/package.py b/scripts/spack_packages/packages/mathpresso/package.py index b050bb6b..c3fa63b8 100644 --- a/scripts/spack_packages/packages/mathpresso/package.py +++ b/scripts/spack_packages/packages/mathpresso/package.py @@ -1,5 +1,6 @@ from spack.package import * import subprocess +from spack_repo.builtin.build_systems.cmake import CMakePackage class Mathpresso(CMakePackage): """MathPresso is a mathematical expression parser and JIT compiler.""" diff --git a/scripts/spack_packages/packages/pygeosx/package.py b/scripts/spack_packages/packages/pygeosx/package.py index aaa1ddc4..acf56fa0 100644 --- a/scripts/spack_packages/packages/pygeosx/package.py +++ b/scripts/spack_packages/packages/pygeosx/package.py @@ -3,7 +3,8 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * +from spack_repo.builtin.build_systems.bundle import BundlePackage class Pygeosx(BundlePackage): """This is a set of libraries necessary for the pygeosx ATS environment. diff --git a/scripts/spack_packages/packages/scotch/package.py b/scripts/spack_packages/packages/scotch/package.py index c79025a8..ace6057a 100644 --- a/scripts/spack_packages/packages/scotch/package.py +++ b/scripts/spack_packages/packages/scotch/package.py @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.build_systems import cmake, makefile -from spack.build_systems.cmake import CMakePackage -from spack.build_systems.makefile import MakefilePackage +from spack_repo.builtin.build_systems import cmake, makefile +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.makefile import MakefilePackage from spack.package import * diff --git a/scripts/spack_packages/packages/trilinos/package.py b/scripts/spack_packages/packages/trilinos/package.py index 20600933..f3f90123 100644 --- a/scripts/spack_packages/packages/trilinos/package.py +++ b/scripts/spack_packages/packages/trilinos/package.py @@ -12,11 +12,12 @@ import os import sys -from spack import * -from spack.build_environment import dso_suffix -from spack.error import NoHeadersError -from spack.operating_systems.mac_os import macos_version -from spack.pkg.builtin.kokkos import Kokkos +from spack.package import * + +from spack_repo.builtin.build_systems.cmake import CMakePackage +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage +from spack_repo.builtin.packages.kokkos.package import Kokkos # Trilinos is complicated to build, as an inspiration a couple of links to # other repositories which build it: @@ -312,11 +313,12 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage): conflicts('gotype=all', when='@12.15:') # CUDA without wrapper requires clang - for _compiler in spack.compilers.supported_compilers(): - if _compiler != 'clang': - conflicts('+cuda', when='~wrapper %' + _compiler, - msg='trilinos~wrapper+cuda can only be built with the ' - 'Clang compiler') + requires( + "%clang", + when="+cuda~wrapper", + msg="trilinos~wrapper+cuda can only be built with the Clang compiler", + ) + conflicts('+cuda_rdc', when='~cuda') conflicts('+rocm_rdc', when='~rocm') conflicts('+wrapper', when='~cuda') @@ -335,6 +337,10 @@ class Trilinos(CMakePackage, CudaPackage, ROCmPackage): conflicts('+stokhos', when='%xl_r') # ###################### Dependencies ########################## + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build", when="+fortran") + depends_on('adios2', when='+adios2') depends_on('blas') diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index 39be1dfb..c75d6cfe 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -6,8 +6,8 @@ import os import sys -from spack import * - +from spack.package import * +from spack_repo.builtin.build_systems.cmake import CMakePackage class Vtk(CMakePackage): """The Visualization Toolkit (VTK) is an open-source, freely @@ -52,7 +52,10 @@ class Vtk(CMakePackage): # We need mpi4py if buidling python wrappers and using MPI depends_on('py-mpi4py', when='+python+mpi', type='run') + depends_on("c", type="build") + depends_on("cxx", type="build") depends_on('mpi', when='+mpi') + patch_dir = os.path.join(os.path.dirname(__file__), '9.4.2-patch') if os.path.isdir(patch_dir): diff --git a/scripts/uberenv b/scripts/uberenv index a76a2942..eb185358 160000 --- a/scripts/uberenv +++ b/scripts/uberenv @@ -1 +1 @@ -Subproject commit a76a2942f0b575f7d9047dfb87d699e2b6d18f12 +Subproject commit eb1853586821360613f1c6c415ebf65d370a47e7