diff --git a/docs/interfaces.md b/docs/interfaces.md index 780041ab..cded8470 100644 --- a/docs/interfaces.md +++ b/docs/interfaces.md @@ -12,7 +12,6 @@ For example, the Spack configuration is in the following files when a stack has ``` /user-environment ├─ config -│ ├─ compilers.yaml │ ├─ repos.yaml │ ├─ packages.yaml │ └─ upstreams.yaml @@ -31,7 +30,6 @@ Notes on the configuration files: system: install_tree: /user-environment ``` -* `compilers.yaml`: includes all compilers that were installed in the `gcc:` and `llvm:` sections of the `compilers.yaml` recipe file. Note that the `bootstrap` compiler is not included. * `packages.yaml`: refers to the external packages that were used to configure the recipe: both the defaults in the cluster configuration, and any additional packages that were set in the recipe. * `repos.yaml`: points to the custom Spack repository: ```yaml diff --git a/stackinator/builder.py b/stackinator/builder.py index 66d50a5d..6f7e6c66 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -266,19 +266,30 @@ def generate(self, recipe): make_user_template = jinja_env.get_template("Make.user") with (self.path / "Make.user").open("w") as f: + base_uenvs = [e["image"] for e in recipe.base_uenv["compilers"]] + if "gpu" in recipe.base_uenv: + base_uenvs += [recipe.base_uenv["gpu"]["image"]] f.write( make_user_template.render( spack_version=spack_version, build_path=self.path, store=recipe.mount, no_bwrap=recipe.no_bwrap, + base_uenv=base_uenvs, verbose=False, ) ) f.write("\n") etc_path = self.root / "etc" - for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py"]: + for f_etc in [ + "Make.inc", + "bwrap-mutable-root.sh", + "bwrap-store.sh", + "envvars.py", + "gen_packages_yaml.py", + "squashfs-mount-wrapper.sh", + ]: shutil.copy2(etc_path / f_etc, self.path / f_etc) # used to configure both pre and post install hooks, if they are provided. @@ -414,7 +425,7 @@ def generate(self, recipe): # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. - repo_dst = store_path / "repo" + repo_dst = store_path / "spack_repo/alps" self._logger.debug(f"creating the stack spack prepo in {repo_dst}") if repo_dst.exists(): self._logger.debug(f"{repo_dst} exists ... deleting") @@ -432,13 +443,14 @@ def generate(self, recipe): """\ repo: namespace: alps + api: v2.0 """ ) # create the repository step 2: create the repos.yaml file in build_path/config repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: - repo_path = recipe.mount / "repo" + repo_path = recipe.mount / "spack_repo/alps" f.write(repos_yaml_template.render(repo_path=repo_path.as_posix(), verbose=False)) f.write("\n") @@ -457,19 +469,6 @@ def generate(self, recipe): elif dst.exists(): self._logger.debug(f" NOT installing package {pkg_path}") - # Generate the makefile and spack.yaml files that describe the compilers - compiler_files = recipe.compiler_files - compiler_path = self.path / "compilers" - compiler_path.mkdir(exist_ok=True) - with (compiler_path / "Makefile").open(mode="w") as f: - f.write(compiler_files["makefile"]) - - for name, yml in compiler_files["config"].items(): - compiler_config_path = compiler_path / name - compiler_config_path.mkdir(exist_ok=True) - with (compiler_config_path / "spack.yaml").open(mode="w") as f: - f.write(yml) - # generate the makefile and spack.yaml files that describe the environments environment_files = recipe.environment_files environments_path = self.path / "environments" @@ -480,6 +479,7 @@ def generate(self, recipe): for name, yml in environment_files["config"].items(): env_config_path = environments_path / name env_config_path.mkdir(exist_ok=True) + # packages.yaml is added in the makefile from the base uenv with (env_config_path / "spack.yaml").open(mode="w") as f: f.write(yml) @@ -490,14 +490,10 @@ def generate(self, recipe): generate_config_path.mkdir(exist_ok=True) # write generate-config/Makefile - all_compilers = [x for x in recipe.compilers.keys()] - release_compilers = [x for x in all_compilers if x != "bootstrap"] with (generate_config_path / "Makefile").open("w") as f: f.write( make_config_template.render( build_path=self.path.as_posix(), - all_compilers=all_compilers, - release_compilers=release_compilers, verbose=False, ) ) diff --git a/stackinator/etc/Make.inc b/stackinator/etc/Make.inc index d677ab13..425456c1 100644 --- a/stackinator/etc/Make.inc +++ b/stackinator/etc/Make.inc @@ -16,7 +16,7 @@ store: mkdir -p $(STORE) # Concretization -%/spack.lock: %/spack.yaml %/compilers.yaml %/config.yaml %/packages.yaml +%/spack.lock: %/spack.yaml %/config.yaml %/packages.yaml $(SPACK_ENV) concretize -f # Generate Makefiles for the environment install diff --git a/stackinator/etc/bwrap-mutable-root.sh b/stackinator/etc/bwrap-mutable-root.sh index 0d360162..908638c8 100755 --- a/stackinator/etc/bwrap-mutable-root.sh +++ b/stackinator/etc/bwrap-mutable-root.sh @@ -1,11 +1,21 @@ #!/bin/bash + +set -euo pipefail args=() shopt -s dotglob + +# from /user-environment/foo/bar/baz store /user-environment as _top_level +_top_level=$(echo $STORE | cut -d "/" -f 2 | xargs printf "/%s") + for d in /*; do + # skip STORE + if [ "$d" = "${_top_level}" ]; then + continue + fi # skip invalid symlinks, as they will break bwrap if [ ! -L "$d" ] || [ -e "$d" ]; then args+=("--dev-bind" "$d" "$d") fi done -PS1="\[\e[36;1m\]build-env >>> \[\e[0m\]" bwrap "${args[@]}" "$@" +PS1="\[\e[36;1m\]build-env >>> \[\e[0m\]" bwrap "${args[@]}" "$@" diff --git a/stackinator/etc/bwrap-store.sh b/stackinator/etc/bwrap-store.sh new file mode 100755 index 00000000..95b86ab4 --- /dev/null +++ b/stackinator/etc/bwrap-store.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +mkdir -p $STORE + +bwrap --dev-bind / / \ + --bind ${BUILD_ROOT}/store $STORE \ + -- "$@" diff --git a/stackinator/etc/gen_packages_yaml.py b/stackinator/etc/gen_packages_yaml.py new file mode 100755 index 00000000..e5a49afe --- /dev/null +++ b/stackinator/etc/gen_packages_yaml.py @@ -0,0 +1,102 @@ +#!/usr/bin/python3 + +import argparse +import json +import subprocess +import pathlib +import yaml + + +def compiler_extra_attributes(name, prefix): + """Find paths to compiler""" + if name == "gcc": + cc = "gcc" + cxx = "g++" + f90 = "gfortran" + elif name == "llvm": + cc = "clang" + cxx = "clang++" + f90 = None + elif name == "nvhpc": + cc = "nvc" + cxx = "nvc++" + f90 = "nvfortran" + else: + # this is not a compiler + return {} + + def find(comp): + p = subprocess.run( + ["find", prefix, "-name", f"{comp}", "-path", "*/bin/*"], + shell=False, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return p.stdout.strip().decode("utf-8") + + extra_attributes = {"extra_attributes": {"compilers": {"c": find(cc), "cxx": find(cxx)}}} + if f90 is not None: + extra_attributes["extra_attributes"]["compilers"]["fortran"] = find(f90) + + return extra_attributes + + +def gen_packages_impl(lock_file, env_path): + spack_lock = json.load(open(lock_file, "r")) + + packages = {"packages": {}} + + for dd in spack_lock["roots"]: + hash = dd["hash"] + # call subprocess to find install dir + spack_find_prefix = subprocess.run( + ["spack", "--color=never", "-e", env_path, "find", "--format={prefix}", f"/{hash}"], + shell=False, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + spack_find_spec = subprocess.run( + ["spack", "--color=never", "-e", env_path, "find", "--format={name}|{version}|{variants}", f"/{hash}"], + shell=False, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + name, version, variants = spack_find_spec.stdout.strip().decode("utf-8").split("|") + prefix = spack_find_prefix.stdout.strip().decode("utf-8") + + packages["packages"][name] = { + "buildable": False, + "externals": [ + { + "spec": f"{name}@{version} {variants}", + "prefix": prefix, + } + ], + } + # add `extra_attributes` for compilers + if name in ["gcc", "nvhpc", "llvm"]: + extra_attributes = compiler_extra_attributes(name, prefix) + packages["packages"][name]["externals"][0].update(extra_attributes) + + return packages + + +if __name__ == "__main__": + # parse CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("--lock-file", help="spack.lock", type=str) + parser.add_argument("--env-path", help="path to spack env", type=str) + parser.add_argument("--view", help="path to spack view", type=str) + + args = parser.parse_args() + + packages = gen_packages_impl(args.lock_file, args.env_path) + + dst = pathlib.Path(args.view) / "packages.yaml" + with open(dst, "w") as f: + yaml.dump(packages, f) diff --git a/stackinator/etc/squashfs-mount-wrapper.sh b/stackinator/etc/squashfs-mount-wrapper.sh new file mode 100755 index 00000000..64167541 --- /dev/null +++ b/stackinator/etc/squashfs-mount-wrapper.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Function to display usage +usage() { + echo "Usage: $0 --build-root= --sqfs-images ': : ..' -- " + exit 1 +} + +# Initialize variables +BUILD_ROOT="" +SQFS_IMAGES="" + +# Parse options using getopt +TEMP=$(getopt -o '' --long build-root: --long sqfs-images: -n "$0" -- "$@") +if [ $? -ne 0 ]; then + echo "Error parsing arguments" >&2 + usage +fi + +# Reset the positional parameters to the short options +eval set -- "$TEMP" + +# Extract options +while true; do + case "$1" in + --build-root) + BUILD_ROOT="$2" + shift 2 + ;; + --sqfs-images) + SQFS_IMAGES="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +if [ -z "$BUILD_ROOT" ]; then + echo "Error: --build-root is required" >&2 + usage +fi + +if [ -z "$SQFS_IMAGES" ]; then + # no images to mount, skip squashfs-mount + exec "$@" +fi + +read -ra array <<<"$SQFS_IMAGES" + +if [ ${#array[@]} -eq 0 ]; then + echo "no mountpoints specified, skip squashfs-mount" + exec "$@" +fi + +build_root_mounts="" +for elem in "${array[@]}"; do + mount_point=${elem#*:} + sqfs=${elem%%:*} + tmp_mount_point="${BUILD_ROOT}/tmp/mounts/${mount_point}" + mkdir -p ${tmp_mount_point} + build_root_mounts="${build_root_mounts} ${sqfs}:${tmp_mount_point}" +done + +squashfs-mount $build_root_mounts -- "$@" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 5f53eb31..70517179 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,6 +3,7 @@ import jinja2 import yaml +import json from . import cache, root_logger, schema, spack_util @@ -15,7 +16,10 @@ class Recipe: "3.0a", "+xpmem fabrics=ch4ofi ch4_max_vcis=4 process_managers=slurm", ), - "openmpi": ("5", "+internal-pmix +legacylaunchers +orterunprefix fabrics=cma,ofi,xpmem schedulers=slurm"), + "openmpi": ( + "5", + "+internal-pmix +legacylaunchers +orterunprefix fabrics=cma,ofi,xpmem schedulers=slurm", + ), } @property @@ -81,17 +85,6 @@ def __init__(self, args): if not self.mount.is_dir(): raise FileNotFoundError(f"the mount point '{self.mount}' must exist") - # required compilers.yaml file - compiler_path = self.path / "compilers.yaml" - self._logger.debug(f"opening {compiler_path}") - if not compiler_path.is_file(): - raise FileNotFoundError(f"The recipe path '{compiler_path}' does not contain compilers.yaml") - - with compiler_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.compilers_validator.validate(raw) - self.generate_compiler_specs(raw) - # required environments.yaml file environments_path = self.path / "environments.yaml" self._logger.debug(f"opening {environments_path}") @@ -100,9 +93,28 @@ def __init__(self, args): with environments_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) + # insert utils env (squashfs) + raw["_internal_utils"] = { + "unify": True, + "mpi": None, + "specs": ["squashfs default_compression=zstd"], + "views": {"_internal_utils": {"link": "roots"}}, + } schema.environments_validator.validate(raw) + self.generate_environment_specs(raw) + # required base-uenv.json file + base_uenv_path = self.path / "base-uenv.json" + self._logger.debug(f"opening {base_uenv_path}") + if base_uenv_path.is_file(): + with base_uenv_path.open() as fid: + raw = json.load(fid) + schema.base_uenv_validator.validate(raw) + self.base_uenv = raw + else: + self.base_uenv = {"compilers": []} + # optional modules.yaml file modules_path = self.path / "modules.yaml" self._logger.debug(f"opening {modules_path}") @@ -303,20 +315,6 @@ def generate_environment_specs(self, raw): # TODO: Create a custom exception type raise Exception(f"Unsupported mpi: {mpi_impl}") - # set constraints that ensure the the main compiler is always used to build packages - # that do not explicitly request a compiler. - for name, config in environments.items(): - compilers = config["compiler"] - if len(compilers) == 1: - config["toolchain_constraints"] = [] - continue - requires = [f"%{compilers[0]['spec']}"] - for spec in config["specs"]: - if "%" in spec: - requires.append(spec) - - config["toolchain_constraints"] = requires - # An awkward hack to work around spack not supporting creating activation # scripts for each file system view in an environment: it only generates them # for the "default" view. @@ -382,72 +380,14 @@ def generate_environment_specs(self, raw): # it separately for configuring the envvars.py helper during the uenv build. extra = view_config.pop("uenv") - environments[cname]["view"] = {"name": view_name, "config": view_config, "extra": extra} + environments[cname]["view"] = { + "name": view_name, + "config": view_config, + "extra": extra, + } self.environments = environments - # creates the self.compilers field that describes the full specifications - # for all of the compilers from the raw compilers.yaml input - def generate_compiler_specs(self, raw): - compilers = {} - - bootstrap = {} - bootstrap["packages"] = { - "external": [ - "perl", - "m4", - "autoconf", - "automake", - "libtool", - "gawk", - "python", - "texinfo", - "gawk", - ], - } - bootstrap_spec = raw["bootstrap"]["spec"] - bootstrap["specs"] = [ - f"{bootstrap_spec} languages=c,c++", - "squashfs default_compression=zstd", - ] - bootstrap["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - compilers["bootstrap"] = bootstrap - - gcc = {} - gcc["packages"] = { - "external": [ - "perl", - "m4", - "autoconf", - "automake", - "libtool", - "gawk", - "python", - "texinfo", - "gawk", - ], - } - gcc["specs"] = raw["gcc"]["specs"] - gcc["requires"] = bootstrap_spec - gcc["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - compilers["gcc"] = gcc - if raw["llvm"] is not None: - llvm = {} - llvm["packages"] = False - llvm["specs"] = [] - for spec in raw["llvm"]["specs"]: - if spec.startswith("nvhpc"): - llvm["specs"].append(f"{spec}~mpi~blas~lapack") - - if spec.startswith("llvm"): - llvm["specs"].append(f"{spec} +clang targets=x86 ~gold ^ninja@kitware") - - llvm["requires"] = raw["llvm"]["requires"] - llvm["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - compilers["llvm"] = llvm - - self.compilers = compilers - # The path of the default configuration for the target system/cluster @property def system_config_path(self): @@ -468,32 +408,6 @@ def system_config_path(self, path): def mount(self): return pathlib.Path(self.config["store"]) - @property - def compiler_files(self): - files = {} - - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(self.template_path), - trim_blocks=True, - lstrip_blocks=True, - ) - - makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None - files["makefile"] = makefile_template.render( - compilers=self.compilers, - push_to_cache=push_to_cache, - spack_version=self.spack_version, - ) - - # generate compilers//spack.yaml - files["config"] = {} - for compiler, config in self.compilers.items(): - spack_yaml_template = env.get_template(f"compilers.{compiler}.spack.yaml") - files["config"][compiler] = spack_yaml_template.render(config=config) - - return files - @property def environment_files(self): files = {} @@ -517,5 +431,16 @@ def environment_files(self): for env, config in self.environments.items(): spack_yaml_template = jenv.get_template("environments.spack.yaml") files["config"][env] = spack_yaml_template.render(config=config, name=env, store=self.mount) - + files_config_env = yaml.safe_load(files["config"][env]) + # add base uenv upstream + for compiler in self.base_uenv["compilers"]: + files_config_env["spack"]["include"] += [ + str(pathlib.Path(compiler["image"]["prefix_path"]) / "env/default/packages.yaml") + ] + # add gpu base uenv + if "gpu" in self.base_uenv: + files_config_env["spack"]["include"] += [ + str(pathlib.Path(self.base_uenv["gpu"]["image"]["prefix_path"]) / "env/default/packages.yaml") + ] + files["config"][env] = yaml.dump(files_config_env) return files diff --git a/stackinator/schema.py b/stackinator/schema.py index 0f6379d6..6d8f8f65 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -52,3 +52,5 @@ def py2yaml(data, indent): environments_validator = validator(environments_schema) cache_schema = json.load(open(prefix / "schema/cache.json")) cache_validator = validator(cache_schema) +base_uenv_schema = json.load(open(prefix / "schema/base-uenv.json")) +base_uenv_validator = validator(base_uenv_schema) diff --git a/stackinator/schema/base-uenv.json b/stackinator/schema/base-uenv.json new file mode 100644 index 00000000..cbf6b2f3 --- /dev/null +++ b/stackinator/schema/base-uenv.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Modular Uenv", + "description": "Schema for modular sqfs", + "type": "object", + "required": ["compilers"], + "defs": { + "image": { + "type": "object", + "properties": { + "sha256": { + "type": "string", + "description": "SHA256 hash of the image", + "pattern": "^[0-9a-f]{64}$" + }, + "file": { + "type": "string", + "description": "File path or name of the image" + }, + "name": { + "type": "string", + "description": "uenv spec" + }, + "prefix_path": { + "type": "string", + "description": "mount point" + } + }, + "anyOf": [{ + "required": ["name"] + }, + { + "required": ["sha256"] + } + ], + "additionalProperties": false + } + }, + "properties": { + "compilers": { + "type": "array", + "description": "List of compiler configurations", + "minItems": 1, + "items": { + "type": "object", + "required": ["type", "version", "image"], + "properties": { + "type": { + "type": "string", + "enum": ["gcc", "llvm", "nvhpc"], + "description": "Type of the compiler" + }, + "image": { + "$ref": "#/defs/image" + } + } + } + }, + "root": { + "type": "object", + "properties": { + "image": { + "$ref": "#/defs/image" + } + }, + "description": "The root sqfs image" + }, + "gpu": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "rocm", + "cuda" + ], + "description": "type of the gpu backend" + }, + "image": { + "$ref": "#/defs/image", + "description": "TOOD" + }, + "required": [ + "type", + "image" + ] + }, + "additionalProperties": false + } + } +} diff --git a/stackinator/schema/compilers.json b/stackinator/schema/compilers.json index 1647a5bd..a618b574 100644 --- a/stackinator/schema/compilers.json +++ b/stackinator/schema/compilers.json @@ -3,7 +3,7 @@ "title": "Schema for Spack Stack compilers.yaml recipe file", "type": "object", "additionalProperties": false, - "required": ["bootstrap", "gcc"], + "required": ["gcc"], "defs": { "gcc_version_spec": { "type": "string", diff --git a/stackinator/schema/environments.json b/stackinator/schema/environments.json index 93c0cf85..ed498b10 100644 --- a/stackinator/schema/environments.json +++ b/stackinator/schema/environments.json @@ -6,7 +6,7 @@ "patternProperties": { "\\w[\\w-]*": { "type": "object", - "required": ["compiler", "specs"], + "required": ["specs"], "additionalProperties": false, "properties": { "deprecated": { @@ -17,17 +17,6 @@ "enum": ["when_possible", true, false], "default": true }, - "compiler": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "toolchain": {"type": "string"}, - "spec": {"type": "string"} - } - } - }, "specs": { "type": "array", "items": {"type": "string"} diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index cb8acb03..3aa3f0aa 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -1,3 +1,4 @@ +{% set spacejoiner = joiner(' ') %} # vim: filetype=make # Copy this file to Make.user and set some variables. @@ -15,6 +16,10 @@ SPACK_HELPER := $(SPACK) --color=never # The Spack installation root. STORE := {{ store }} +# export as env vars (required by bwrap scripts) +export STORE := $(STORE) +export BUILD_ROOT := $(BUILD_ROOT) + # When already building inside a sandbox, use `SANDBOX :=` (empty string) # Without a sandbox, make sure to hide sensitive data such as ~/.ssh through bubblewrap. # Also bind the directories `./tmp -> /tmp` and `./store -> $(STORE)`, so that @@ -22,13 +27,22 @@ STORE := {{ store }} # put the project itself in-memory, or use a flag like --bind /dev/shm/store # $(STORE). Use `bwrap-mutable-root.sh` in case you need to create a new # directory at the root /. +SQFS_MOUNT := $(BUILD_ROOT)/squashfs-mount-wrapper.sh --build-root=$(BUILD_ROOT) --sqfs-images="{% for item in base_uenv %}{{ spacejoiner() }}{{ item.file }}:{{item.prefix_path}}{% endfor %}" {% if no_bwrap %} -SANDBOX := +SANDBOX := $SQFS_MOUNT -- {% else %} -SANDBOX := $(BUILD_ROOT)/bwrap-mutable-root.sh $\ +SANDBOX := $(SQFS_MOUNT) -- $(BUILD_ROOT)/bwrap-mutable-root.sh $\ + --tmpfs ~ $\ + --bind $(BUILD_ROOT)/tmp /tmp $\ + --bind $(BUILD_ROOT)/store $(STORE) $\ +{% for item in base_uenv %} + --bind $(BUILD_ROOT)/tmp/mounts/{{item.prefix_path}} {{item.prefix_path}} $\ +{% endfor %} + +SANDBOX_NO_BASE := $(BUILD_ROOT)/bwrap-mutable-root.sh $\ --tmpfs ~ $\ --bind $(BUILD_ROOT)/tmp /tmp $\ - --bind $(BUILD_ROOT)/store $(STORE) + -- $(BUILD_ROOT)/bwrap-store.sh {% endif %} # Makes sure that make -Orecurse continues to print in color. export SPACK_COLOR := always diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index c16df68d..46c6dfbc 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -51,13 +51,10 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} touch mirror-setup -compilers: mirror-setup +generate-config: mirror-setup $(SANDBOX) $(MAKE) -C $@ -generate-config: compilers - $(SANDBOX) $(MAKE) -C $@ - -environments: compilers +environments: mirror-setup $(SANDBOX) $(MAKE) -C $@ {% if modules %} @@ -81,9 +78,9 @@ post-install: env-meta # Create a squashfs file from the installed software. store.squashfs: post-install # clean up the __pycache__ paths in the repo - $(SANDBOX) find $(STORE)/repo -type d -name __pycache__ -exec rm -r {} + - $(SANDBOX) chmod -R a+rX $(STORE) - $(SANDBOX) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -e ./compilers/bootstrap find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -force-uid nobody -force-gid nobody -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 + $(SANDBOX_NO_BASE) find $(STORE)/spack_repo -type d -name __pycache__ -exec rm -r {} + + $(SANDBOX_NO_BASE) chmod -R a+rX $(STORE) + $(SANDBOX_NO_BASE) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -force-uid nobody -force-gid nobody -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 # Force push all built packages to the build cache cache-force: mirror-setup @@ -110,6 +107,6 @@ build.tar.gz: spack-version Make.user Make.inc Makefile | environments # Clean generate files, does *not* remove installed software. clean: - rm -rf -- $(wildcard */*/spack.lock) $(wildcard */*/.spack-env) $(wildcard */*/Makefile) $(wildcard */*/generated) $(wildcard cache) $(wildcard compilers/*/config.yaml) $(wildcard compilers/*/packages.yaml) $(wildcard compilers/*/compilers.yaml) $(wildcard environments/*/config.yaml) $(wildcard environments/*/packages.yaml) $(wildcard environments/*/compilers.yaml) post-install modules-done env-meta store.squashfs + rm -rf -- $(wildcard */*/spack.lock) $(wildcard */*/.spack-env) $(wildcard */*/Makefile) $(wildcard */*/generated) $(wildcard cache) $(wildcard compilers/*/config.yaml) $(wildcard compilers/*/packages.yaml) $(wildcard compilers/*/compilers.yaml) $(wildcard environments/*/config.yaml) $(wildcard environments/*/packages.yaml) post-install modules-done env-meta store.squashfs include Make.inc diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers deleted file mode 100644 index 37432849..00000000 --- a/stackinator/templates/Makefile.compilers +++ /dev/null @@ -1,72 +0,0 @@ -{% set pipejoiner = joiner('|') %} --include ../Make.user - -MAKEFLAGS += --output-sync=recurse - -.PHONY: all .locks .packages.yaml - -all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfor %} - - -# Ensure that spack.lock files are never removed as intermediate files... -.locks:{% for compiler in compilers %} {{ compiler }}/spack.lock{% endfor %} - - -# Ensure that package yaml files are never removed as intermediate files... -.packages.yaml:{% for compiler in compilers %} {{ compiler }}/packages.yaml{% endfor %} - - -{% for compiler, config in compilers.items() %} -{{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ - $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ - | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | cut -d ';' -f2) -{% endif %} - touch $@ - -{% endfor %} - -# Configure the install location. -{% for compiler in compilers %}{{ compiler }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - -# Configure external system dependencies for each compiler toolchain -{% for compiler, config in compilers.items() %} -{% if config.packages and config.packages.external %} -{{ compiler }}/packages.yaml: - $(SPACK) external find --scope=user {% for package in config.packages.external %} {{package}}{% endfor %} - -{% endif %} -{% endfor %} -# Configure dependencies between compilers -gcc/compilers.yaml: bootstrap/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./bootstrap find --format '{prefix}' {{ compilers.gcc.requires }})) - -{% if compilers.llvm %} -llvm/compilers.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --format '{prefix}' {{ compilers.llvm.requires }})) -{% endif %} - - -include ../Make.inc - -# GNU Make isn't very smart about dependencies across included Makefiles, so we -# specify the order here by conditionally including them, when the dependent exists. -ifeq (,$(filter clean,$(MAKECMDGOALS))) - -include bootstrap/Makefile - -ifneq (,$(wildcard bootstrap/Makefile)) -include gcc/Makefile -endif - -{% if compilers.llvm %} -ifneq (,$(wildcard gcc/Makefile)) -include llvm/Makefile -endif -{% endif %} - - -endif diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5f163710..13bf577b 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -33,7 +33,8 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {{ env }}/generated/view_config: {{ env }}/generated/env {% if config.view %} $(SPACK) env activate --with-view default --sh ./{{ env }} > $(STORE)/env/{{ config.view.name }}/activate.sh - $(BUILD_ROOT)/envvars.py view {% if config.view.extra.add_compilers %}--compilers=./{{ env }}/compilers.yaml {% endif %} --prefix_paths="{{ config.view.extra.prefix_string }}" $(STORE)/env/{{ config.view.name }} $(BUILD_ROOT) + $(BUILD_ROOT)/envvars.py view --prefix_paths="{{ config.view.extra.prefix_string }}" $(STORE)/env/{{ config.view.name }} $(BUILD_ROOT) + $(BUILD_ROOT)/gen_packages_yaml.py --view=$(STORE)/env/{{ config.view.name }} --env=./{{ env }} --lock-file=./{{ env }}/spack.lock {% endif %} touch $@ @@ -43,15 +44,6 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {% for env in environments %}{{ env }}/config.yaml {% endfor %}: | store $(SPACK) config --scope=user add config:install_tree:root:$(STORE) -# Create the compilers.yaml configuration for each environment -{% for env, config in environments.items() %} -{{ env }}_PREFIX = {% for C in config.compiler %} $$($(SPACK_HELPER) -e ../compilers/{{ C.toolchain }} find --format '{prefix}' {{ C.spec }}){% endfor %} - -{{ env }}/compilers.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $({{ env }}_PREFIX)) - -{% endfor %} - # Configure external system dependencies for each compiler toolchain {% for env, config in environments.items() %} {% if config.packages %} diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index cb6c8528..c726903e 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -3,35 +3,22 @@ include ../Make.user CONFIG_DIR = $(STORE)/config MODULE_DIR = $(BUILD_ROOT)/modules -# These will be the prefixes of the GCCs, LLVMs and NVHPCs in the respective environments. -ALL_COMPILER_PREFIXES ={% for compiler in all_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} +# COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} -COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} +all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml -all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/compilers.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml $(MODULE_DIR)/compilers.yaml - -# Generate the upstream configuration that will be provided by the mounted image -$(CONFIG_DIR)/compilers.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(COMPILER_PREFIXES)) - $(CONFIG_DIR)/upstreams.yaml: $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) # Copy the cluster-specific packages.yaml file to the configuration. # requires compilers.yaml to ensure that the path $(CONFIG_DIR) has been created. -$(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/compilers.yaml - install -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml - -$(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/compilers.yaml - install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml +$(CONFIG_DIR)/packages.yaml: + install -D -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml -# Generate a configuration used to generate the module files -# The configuration in CONFIG_DIR can't be used for this purpose, because a compilers.yaml -# that includes the bootstrap compiler is required to build the modules. -$(MODULE_DIR)/compilers.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(ALL_COMPILER_PREFIXES)) +$(CONFIG_DIR)/repos.yaml: + install -D -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml: $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) diff --git a/stackinator/templates/compilers.bootstrap.spack.yaml b/stackinator/templates/compilers.bootstrap.spack.yaml deleted file mode 100644 index 88c0b541..00000000 --- a/stackinator/templates/compilers.bootstrap.spack.yaml +++ /dev/null @@ -1,26 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: true - reuse: false - packages: - gcc: - variants: [build_type=Release ~bootstrap +strip] - mpc: - variants: [libs=static] - gmp: - variants: [libs=static] - mpfr: - variants: [libs=static] - zstd: - variants: [libs=static] - zlib: - variants: [~shared] - diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml deleted file mode 100644 index 44cf0828..00000000 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ /dev/null @@ -1,26 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - - compilers.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - packages: - gcc: - variants: [build_type=Release +bootstrap +strip] - mpc: - variants: [libs=static] - gmp: - variants: [libs=static] - mpfr: - variants: [libs=static] - zstd: - variants: [libs=static] - zlib: - variants: [~shared] diff --git a/stackinator/templates/compilers.llvm.spack.yaml b/stackinator/templates/compilers.llvm.spack.yaml deleted file mode 100644 index c7882ed0..00000000 --- a/stackinator/templates/compilers.llvm.spack.yaml +++ /dev/null @@ -1,14 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - - compilers.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - diff --git a/stackinator/templates/environments.spack.yaml b/stackinator/templates/environments.spack.yaml index 2477eea6..27287215 100644 --- a/stackinator/templates/environments.spack.yaml +++ b/stackinator/templates/environments.spack.yaml @@ -3,7 +3,6 @@ spack: {% if config.packages %} - packages.yaml {% endif %} - - compilers.yaml - config.yaml config: deprecated: {{ config.deprecated }} @@ -14,23 +13,25 @@ spack: {% for spec in config.specs %} - '{{ spec }}' {% endfor %} +{% if config.toolchain_constraints or config.variants or config.mpi.spec %} packages: + {% if config.toolchain_constraints or config.variants %} all: -{% set separator = joiner(', ') %} - compiler: [{% for c in config.compiler %}{{ separator() }}'{{ c.spec }}'{% endfor %}] + {% endif %} {% if config.toolchain_constraints %} require: {% set separator = joiner(', ') %} - one_of: [{% for c in config.toolchain_constraints %}{{ separator() }}'{{ c }}'{% endfor %}] {% endif %} - {% if config.variants %} +{% if config.variants %} {% set separator = joiner(', ') %} variants: [{% for v in config.variants %}{{ separator() }}'{{ v }}'{% endfor %}] - {% endif %} - {% if config.mpi.spec %} +{% endif %} +{% if config.mpi.spec %} mpi: require: '{{ config.mpi.spec }}' - {% endif %} +{% endif %} +{% endif %} {% if config.view %} view: default: diff --git a/stackinator/templates/stack-debug.sh b/stackinator/templates/stack-debug.sh old mode 100644 new mode 100755 index 4c897640..ee442cae --- a/stackinator/templates/stack-debug.sh +++ b/stackinator/templates/stack-debug.sh @@ -1 +1,14 @@ -env --ignore-environment PATH=/usr/bin:/bin:{{ build_path }}/spack/bin HOME=$HOME BUILD_ROOT={{ build_path }} STORE={{ mount_path }} SPACK_SYSTEM_CONFIG_PATH={{ build_path }}/config SPACK_USER_CACHE_PATH={{ build_path }}/cache SPACK=spack SPACK_COLOR=always SPACK_USER_CONFIG_PATH=~ LC_ALL=en_US.UTF-8 TZ=UTC SOURCE_DATE_EPOCH=315576060 {% if use_bwrap %} {{ build_path }}/bwrap-mutable-root.sh --tmpfs ~ --bind {{ build_path }}/tmp /tmp --bind {{ build_path }}/store {{ mount_path }} {% endif %} bash -noprofile -l +#!/bin/bash + +set -eu + +env --ignore-environment \ + PATH=/usr/bin:/bin:{{ build_path }}/spack/bin \ + HOME=$HOME BUILD_ROOT={{ build_path }} \ + STORE={{ mount_path }} SPACK_SYSTEM_CONFIG_PATH={{ build_path }}/config \ + SPACK_USER_CACHE_PATH={{ build_path }}/cache \ + SPACK=spack SPACK_COLOR=always \ + SPACK_USER_CONFIG_PATH={% if spack_version>="0.23" %}~{% else %}/dev/null{% endif %} \ + LC_ALL=en_US.UTF-8 TZ=UTC SOURCE_DATE_EPOCH=315576060 \ + {% if use_bwrap %} {{ build_path }}/bwrap-mutable-root.sh --tmpfs ~ --bind {{ build_path }}/tmp /tmp -- {{ build_path }}/bwrap-store.sh {% endif %} \ + bash -noprofile -l