From 97716ec7ec6bc1090144859334b59df8f0934fb9 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Mon, 24 Feb 2025 22:34:10 +0100 Subject: [PATCH 1/8] typo --- stackinator/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 66293147..fd8ac663 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -402,7 +402,7 @@ def generate_environment_specs(self, raw): self.environments = environments # creates the self.compilers field that describes the full specifications - # for all of teh compilers from the raw compilers.yaml input + # for all of the compilers from the raw compilers.yaml input def generate_compiler_specs(self, raw): compilers = {} From eff11274b1a9dc19f1309dcfddfcb7dffcbe4479 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Tue, 25 Feb 2025 12:46:54 +0100 Subject: [PATCH 2/8] remove duplicate target --- stackinator/templates/Makefile.compilers | 4 ---- 1 file changed, 4 deletions(-) diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index 631c110b..f32eb0e5 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -28,10 +28,6 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% endfor %} -# Configure the install location. -{% for compiler in compilers %}{{ compiler }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - # Configure the install location. {% for compiler in compilers %}{{ compiler }}/config.yaml {% endfor %}: | store $(SPACK) config --scope=user add config:install_tree:root:$(STORE) From b99664b96962bff944ae98590bbf274d0d57db0e Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Tue, 25 Feb 2025 13:50:31 +0100 Subject: [PATCH 3/8] modular uenv remove bootstrap from schema remove compilers.yaml add squashfs-mount for base-uenvs cleanup cleanup change order typo typo typo skip compiler target convert to string fix path skip base uenvs in post-install loop.last tab look for squashfs in global tree always install squashfs add dependency do not generate {{env}}/compilers.yaml skip {{env}}/compilers.yaml in envvars.py skip -C /user-environment/config do not find compilers cleanup deps remove compilers.yaml install squashfs to bind mounted /tmp do not use filesystem locks remove dependency remove -d install squashfs in an env fix fix cleanup install -D the mount point doesn't need to exist base-uenv.json is optional wip fix squashfs-mount-wrapper allow to use a non-existent directory for STORE comment add padded_length for cache add missing file undo use padded_lenght in env do not use self.mount for mirror path use same sandbox in stack-debug.sh Revert "do not use self.mount for mirror path" This reverts commit f187450c97cdd65791abd5173f481bf58ca53c0c. ignore store/mountpath for build cache fix stack-debug.sh fix bwrap-store Revert "ignore store/mountpath for build cache" This reverts commit a13fc2b06e28e2ed9604d657982b97ba505fb1a9. remove padded_length remove stdout add generation of packages.yaml for externals fix chmod wip avoid spack --color fix structure set buildable false fix gen packages.yaml update do not use -all-root remove unused compiler target v2.0 repo format path must contain spack_repo repo -> spack_repo repo path must include namespace repo -> spack_repo update base-uenv.json schema compilers.yaml -> packages.yaml fix sandbox fix path, version doesn't belong here add missing version to packages.yaml set extra_attributes for compilers fix fix split debug Revert "debug" This reverts commit 4172a0a3cdd5d7a7fff73a961d542f4be4182e48. fix find command cleanup update the schema remove compilers.yaml and the bits related to it from recipe.py remove more occurences of compilers.yaml avoid empty packages.all in environments.spack.yaml update for new base_uenv.json schema typo in schema base_uenv schema, default null for optional entries fix schema wip schema fix schema typo fix environments.spack.yaml typo fix environment.spack.yaml include exports from the gpu sub uenv fix schema change title remove compiler from env schema remove compiler from _internal_utils --- docs/interfaces.md | 2 - stackinator/builder.py | 36 ++-- stackinator/etc/Make.inc | 2 +- stackinator/etc/bwrap-mutable-root.sh | 12 +- stackinator/etc/bwrap-store.sh | 9 + stackinator/etc/gen_packages_yaml.py | 102 ++++++++++ stackinator/etc/squashfs-mount-wrapper.sh | 70 +++++++ stackinator/recipe.py | 176 +++++------------- stackinator/schema.py | 2 + stackinator/schema/base-uenv.json | 91 +++++++++ stackinator/schema/compilers.json | 2 +- stackinator/schema/environments.json | 13 +- stackinator/templates/Make.user | 20 +- stackinator/templates/Makefile | 16 +- stackinator/templates/Makefile.compilers | 23 +-- stackinator/templates/Makefile.environments | 16 +- .../templates/Makefile.generate-config | 26 ++- .../templates/compilers.bootstrap.spack.yaml | 26 --- .../templates/compilers.gcc.spack.yaml | 1 - .../templates/compilers.llvm.spack.yaml | 1 - stackinator/templates/environments.spack.yaml | 15 +- stackinator/templates/stack-debug.sh | 15 +- 22 files changed, 429 insertions(+), 247 deletions(-) create mode 100755 stackinator/etc/bwrap-store.sh create mode 100755 stackinator/etc/gen_packages_yaml.py create mode 100755 stackinator/etc/squashfs-mount-wrapper.sh create mode 100644 stackinator/schema/base-uenv.json delete mode 100644 stackinator/templates/compilers.bootstrap.spack.yaml diff --git a/docs/interfaces.md b/docs/interfaces.md index 780041ab..cded8470 100644 --- a/docs/interfaces.md +++ b/docs/interfaces.md @@ -12,7 +12,6 @@ For example, the Spack configuration is in the following files when a stack has ``` /user-environment ├─ config -│ ├─ compilers.yaml │ ├─ repos.yaml │ ├─ packages.yaml │ └─ upstreams.yaml @@ -31,7 +30,6 @@ Notes on the configuration files: system: install_tree: /user-environment ``` -* `compilers.yaml`: includes all compilers that were installed in the `gcc:` and `llvm:` sections of the `compilers.yaml` recipe file. Note that the `bootstrap` compiler is not included. * `packages.yaml`: refers to the external packages that were used to configure the recipe: both the defaults in the cluster configuration, and any additional packages that were set in the recipe. * `repos.yaml`: points to the custom Spack repository: ```yaml diff --git a/stackinator/builder.py b/stackinator/builder.py index 61d0e3ad..b08c6e00 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -266,19 +266,30 @@ def generate(self, recipe): make_user_template = jinja_env.get_template("Make.user") with (self.path / "Make.user").open("w") as f: + base_uenvs = [e["image"] for e in recipe.base_uenv["compilers"]] + if "gpu" in recipe.base_uenv: + base_uenvs += [recipe.base_uenv["gpu"]["image"]] f.write( make_user_template.render( spack_version=spack_version, build_path=self.path, store=recipe.mount, no_bwrap=recipe.no_bwrap, + base_uenv=base_uenvs, verbose=False, ) ) f.write("\n") etc_path = self.root / "etc" - for f_etc in ["Make.inc", "bwrap-mutable-root.sh", "envvars.py"]: + for f_etc in [ + "Make.inc", + "bwrap-mutable-root.sh", + "bwrap-store.sh", + "envvars.py", + "gen_packages_yaml.py", + "squashfs-mount-wrapper.sh", + ]: shutil.copy2(etc_path / f_etc, self.path / f_etc) # used to configure both pre and post install hooks, if they are provided. @@ -414,7 +425,7 @@ def generate(self, recipe): # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. - repo_dst = store_path / "repo" + repo_dst = store_path / "spack_repo/alps" self._logger.debug(f"creating the stack spack prepo in {repo_dst}") if repo_dst.exists(): self._logger.debug(f"{repo_dst} exists ... deleting") @@ -432,13 +443,14 @@ def generate(self, recipe): """\ repo: namespace: alps + api: v2.0 """ ) # create the repository step 2: create the repos.yaml file in build_path/config repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: - repo_path = recipe.mount / "repo" + repo_path = recipe.mount / "spack_repo/alps" f.write(repos_yaml_template.render(repo_path=repo_path.as_posix(), verbose=False)) f.write("\n") @@ -457,19 +469,6 @@ def generate(self, recipe): elif dst.exists(): self._logger.debug(f" NOT installing package {pkg_path}") - # Generate the makefile and spack.yaml files that describe the compilers - compiler_files = recipe.compiler_files - compiler_path = self.path / "compilers" - compiler_path.mkdir(exist_ok=True) - with (compiler_path / "Makefile").open(mode="w") as f: - f.write(compiler_files["makefile"]) - - for name, yml in compiler_files["config"].items(): - compiler_config_path = compiler_path / name - compiler_config_path.mkdir(exist_ok=True) - with (compiler_config_path / "spack.yaml").open(mode="w") as f: - f.write(yml) - # generate the makefile and spack.yaml files that describe the environments environment_files = recipe.environment_files environments_path = self.path / "environments" @@ -480,6 +479,7 @@ def generate(self, recipe): for name, yml in environment_files["config"].items(): env_config_path = environments_path / name env_config_path.mkdir(exist_ok=True) + # packages.yaml is added in the makefile from the base uenv with (env_config_path / "spack.yaml").open(mode="w") as f: f.write(yml) @@ -490,14 +490,10 @@ def generate(self, recipe): generate_config_path.mkdir(exist_ok=True) # write generate-config/Makefile - all_compilers = [x for x in recipe.compilers.keys()] - release_compilers = [x for x in all_compilers if x != "bootstrap"] with (generate_config_path / "Makefile").open("w") as f: f.write( make_config_template.render( build_path=self.path.as_posix(), - all_compilers=all_compilers, - release_compilers=release_compilers, verbose=False, ) ) diff --git a/stackinator/etc/Make.inc b/stackinator/etc/Make.inc index d677ab13..425456c1 100644 --- a/stackinator/etc/Make.inc +++ b/stackinator/etc/Make.inc @@ -16,7 +16,7 @@ store: mkdir -p $(STORE) # Concretization -%/spack.lock: %/spack.yaml %/compilers.yaml %/config.yaml %/packages.yaml +%/spack.lock: %/spack.yaml %/config.yaml %/packages.yaml $(SPACK_ENV) concretize -f # Generate Makefiles for the environment install diff --git a/stackinator/etc/bwrap-mutable-root.sh b/stackinator/etc/bwrap-mutable-root.sh index 0d360162..908638c8 100755 --- a/stackinator/etc/bwrap-mutable-root.sh +++ b/stackinator/etc/bwrap-mutable-root.sh @@ -1,11 +1,21 @@ #!/bin/bash + +set -euo pipefail args=() shopt -s dotglob + +# from /user-environment/foo/bar/baz store /user-environment as _top_level +_top_level=$(echo $STORE | cut -d "/" -f 2 | xargs printf "/%s") + for d in /*; do + # skip STORE + if [ "$d" = "${_top_level}" ]; then + continue + fi # skip invalid symlinks, as they will break bwrap if [ ! -L "$d" ] || [ -e "$d" ]; then args+=("--dev-bind" "$d" "$d") fi done -PS1="\[\e[36;1m\]build-env >>> \[\e[0m\]" bwrap "${args[@]}" "$@" +PS1="\[\e[36;1m\]build-env >>> \[\e[0m\]" bwrap "${args[@]}" "$@" diff --git a/stackinator/etc/bwrap-store.sh b/stackinator/etc/bwrap-store.sh new file mode 100755 index 00000000..95b86ab4 --- /dev/null +++ b/stackinator/etc/bwrap-store.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +mkdir -p $STORE + +bwrap --dev-bind / / \ + --bind ${BUILD_ROOT}/store $STORE \ + -- "$@" diff --git a/stackinator/etc/gen_packages_yaml.py b/stackinator/etc/gen_packages_yaml.py new file mode 100755 index 00000000..e5a49afe --- /dev/null +++ b/stackinator/etc/gen_packages_yaml.py @@ -0,0 +1,102 @@ +#!/usr/bin/python3 + +import argparse +import json +import subprocess +import pathlib +import yaml + + +def compiler_extra_attributes(name, prefix): + """Find paths to compiler""" + if name == "gcc": + cc = "gcc" + cxx = "g++" + f90 = "gfortran" + elif name == "llvm": + cc = "clang" + cxx = "clang++" + f90 = None + elif name == "nvhpc": + cc = "nvc" + cxx = "nvc++" + f90 = "nvfortran" + else: + # this is not a compiler + return {} + + def find(comp): + p = subprocess.run( + ["find", prefix, "-name", f"{comp}", "-path", "*/bin/*"], + shell=False, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return p.stdout.strip().decode("utf-8") + + extra_attributes = {"extra_attributes": {"compilers": {"c": find(cc), "cxx": find(cxx)}}} + if f90 is not None: + extra_attributes["extra_attributes"]["compilers"]["fortran"] = find(f90) + + return extra_attributes + + +def gen_packages_impl(lock_file, env_path): + spack_lock = json.load(open(lock_file, "r")) + + packages = {"packages": {}} + + for dd in spack_lock["roots"]: + hash = dd["hash"] + # call subprocess to find install dir + spack_find_prefix = subprocess.run( + ["spack", "--color=never", "-e", env_path, "find", "--format={prefix}", f"/{hash}"], + shell=False, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + spack_find_spec = subprocess.run( + ["spack", "--color=never", "-e", env_path, "find", "--format={name}|{version}|{variants}", f"/{hash}"], + shell=False, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + name, version, variants = spack_find_spec.stdout.strip().decode("utf-8").split("|") + prefix = spack_find_prefix.stdout.strip().decode("utf-8") + + packages["packages"][name] = { + "buildable": False, + "externals": [ + { + "spec": f"{name}@{version} {variants}", + "prefix": prefix, + } + ], + } + # add `extra_attributes` for compilers + if name in ["gcc", "nvhpc", "llvm"]: + extra_attributes = compiler_extra_attributes(name, prefix) + packages["packages"][name]["externals"][0].update(extra_attributes) + + return packages + + +if __name__ == "__main__": + # parse CLI arguments + parser = argparse.ArgumentParser() + parser.add_argument("--lock-file", help="spack.lock", type=str) + parser.add_argument("--env-path", help="path to spack env", type=str) + parser.add_argument("--view", help="path to spack view", type=str) + + args = parser.parse_args() + + packages = gen_packages_impl(args.lock_file, args.env_path) + + dst = pathlib.Path(args.view) / "packages.yaml" + with open(dst, "w") as f: + yaml.dump(packages, f) diff --git a/stackinator/etc/squashfs-mount-wrapper.sh b/stackinator/etc/squashfs-mount-wrapper.sh new file mode 100755 index 00000000..64167541 --- /dev/null +++ b/stackinator/etc/squashfs-mount-wrapper.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Function to display usage +usage() { + echo "Usage: $0 --build-root= --sqfs-images ': : ..' -- " + exit 1 +} + +# Initialize variables +BUILD_ROOT="" +SQFS_IMAGES="" + +# Parse options using getopt +TEMP=$(getopt -o '' --long build-root: --long sqfs-images: -n "$0" -- "$@") +if [ $? -ne 0 ]; then + echo "Error parsing arguments" >&2 + usage +fi + +# Reset the positional parameters to the short options +eval set -- "$TEMP" + +# Extract options +while true; do + case "$1" in + --build-root) + BUILD_ROOT="$2" + shift 2 + ;; + --sqfs-images) + SQFS_IMAGES="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +if [ -z "$BUILD_ROOT" ]; then + echo "Error: --build-root is required" >&2 + usage +fi + +if [ -z "$SQFS_IMAGES" ]; then + # no images to mount, skip squashfs-mount + exec "$@" +fi + +read -ra array <<<"$SQFS_IMAGES" + +if [ ${#array[@]} -eq 0 ]; then + echo "no mountpoints specified, skip squashfs-mount" + exec "$@" +fi + +build_root_mounts="" +for elem in "${array[@]}"; do + mount_point=${elem#*:} + sqfs=${elem%%:*} + tmp_mount_point="${BUILD_ROOT}/tmp/mounts/${mount_point}" + mkdir -p ${tmp_mount_point} + build_root_mounts="${build_root_mounts} ${sqfs}:${tmp_mount_point}" +done + +squashfs-mount $build_root_mounts -- "$@" diff --git a/stackinator/recipe.py b/stackinator/recipe.py index fd8ac663..a55b37d1 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,6 +3,7 @@ import jinja2 import yaml +import json from . import cache, root_logger, schema, spack_util @@ -55,20 +56,9 @@ def __init__(self, args): if args.mount: self.config["store"] = args.mount - # ensure that the requested mount point exists - if not self.mount.is_dir(): - raise FileNotFoundError(f"the mount point '{self.mount}' must exist") - - # required compiler.yaml file - compiler_path = self.path / "compilers.yaml" - self._logger.debug(f"opening {compiler_path}") - if not compiler_path.is_file(): - raise FileNotFoundError(f"The recipe path '{compiler_path}' does " f"not contain compilers.yaml") - - with compiler_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.compilers_validator.validate(raw) - self.generate_compiler_specs(raw) + # # ensure that the requested mount point exists + # if not self.mount.is_dir(): + # raise FileNotFoundError(f"the mount point '{self.mount}' must exist") # required environments.yaml file environments_path = self.path / "environments.yaml" @@ -78,9 +68,28 @@ def __init__(self, args): with environments_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) + # insert utils env (squashfs) + raw["_internal_utils"] = { + "unify": True, + "mpi": None, + "specs": ["squashfs default_compression=zstd"], + "views": {"_internal_utils": {"link": "roots"}}, + } schema.environments_validator.validate(raw) + self.generate_environment_specs(raw) + # required base-uenv.json file + base_uenv_path = self.path / "base-uenv.json" + self._logger.debug(f"opening {base_uenv_path}") + if base_uenv_path.is_file(): + with base_uenv_path.open() as fid: + raw = json.load(fid) + schema.base_uenv_validator.validate(raw) + self.base_uenv = raw + else: + self.base_uenv = {"compilers": []} + # optional modules.yaml file modules_path = self.path / "modules.yaml" self._logger.debug(f"opening {modules_path}") @@ -320,17 +329,17 @@ def generate_environment_specs(self, raw): # set constraints that ensure the the main compiler is always used to build packages # that do not explicitly request a compiler. - for name, config in environments.items(): - compilers = config["compiler"] - if len(compilers) == 1: - config["toolchain_constraints"] = [] - continue - requires = [f"%{compilers[0]['spec']}"] - for spec in config["specs"]: - if "%" in spec: - requires.append(spec) - - config["toolchain_constraints"] = requires + # for name, config in environments.items(): + # compilers = config["compiler"] + # if len(compilers) == 1: + # config["toolchain_constraints"] = [] + # continue + # requires = [f"%{compilers[0]['spec']}"] + # for spec in config["specs"]: + # if "%" in spec: + # requires.append(spec) + + # config["toolchain_constraints"] = requires # An awkward hack to work around spack not supporting creating activation # scripts for each file system view in an environment: it only generates them @@ -401,84 +410,6 @@ def generate_environment_specs(self, raw): self.environments = environments - # creates the self.compilers field that describes the full specifications - # for all of the compilers from the raw compilers.yaml input - def generate_compiler_specs(self, raw): - compilers = {} - - bootstrap = {} - bootstrap["packages"] = { - "external": [ - "perl", - "m4", - "autoconf", - "automake", - "libtool", - "gawk", - "python", - "texinfo", - "gawk", - ], - "variants": { - "gcc": "[build_type=Release ~bootstrap +strip]", - "mpc": "[libs=static]", - "gmp": "[libs=static]", - "mpfr": "[libs=static]", - "zstd": "[libs=static]", - "zlib": "[~shared]", - }, - } - bootstrap_spec = raw["bootstrap"]["spec"] - bootstrap["specs"] = [ - f"{bootstrap_spec} languages=c,c++", - "squashfs default_compression=zstd", - ] - bootstrap["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - compilers["bootstrap"] = bootstrap - - gcc = {} - gcc["packages"] = { - "external": [ - "perl", - "m4", - "autoconf", - "automake", - "libtool", - "gawk", - "python", - "texinfo", - "gawk", - ], - "variants": { - "gcc": "[build_type=Release +profiled +strip]", - "mpc": "[libs=static]", - "gmp": "[libs=static]", - "mpfr": "[libs=static]", - "zstd": "[libs=static]", - "zlib": "[~shared]", - }, - } - gcc["specs"] = raw["gcc"]["specs"] - gcc["requires"] = bootstrap_spec - gcc["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - compilers["gcc"] = gcc - if raw["llvm"] is not None: - llvm = {} - llvm["packages"] = False - llvm["specs"] = [] - for spec in raw["llvm"]["specs"]: - if spec.startswith("nvhpc"): - llvm["specs"].append(f"{spec}~mpi~blas~lapack") - - if spec.startswith("llvm"): - llvm["specs"].append(f"{spec} +clang targets=x86 ~gold ^ninja@kitware") - - llvm["requires"] = raw["llvm"]["requires"] - llvm["exclude_from_cache"] = ["cuda", "nvhpc", "perl"] - compilers["llvm"] = llvm - - self.compilers = compilers - # The path of the default configuration for the target system/cluster @property def system_config_path(self): @@ -499,32 +430,6 @@ def system_config_path(self, path): def mount(self): return pathlib.Path(self.config["store"]) - @property - def compiler_files(self): - files = {} - - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(self.template_path), - trim_blocks=True, - lstrip_blocks=True, - ) - - makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None - files["makefile"] = makefile_template.render( - compilers=self.compilers, - push_to_cache=push_to_cache, - spack_version=self.spack_version, - ) - - # generate compilers//spack.yaml - files["config"] = {} - for compiler, config in self.compilers.items(): - spack_yaml_template = env.get_template(f"compilers.{compiler}.spack.yaml") - files["config"][compiler] = spack_yaml_template.render(config=config) - - return files - @property def environment_files(self): files = {} @@ -548,5 +453,16 @@ def environment_files(self): for env, config in self.environments.items(): spack_yaml_template = jenv.get_template("environments.spack.yaml") files["config"][env] = spack_yaml_template.render(config=config, name=env, store=self.mount) - + files_config_env = yaml.safe_load(files["config"][env]) + # add base uenv upstream + for compiler in self.base_uenv["compilers"]: + files_config_env["spack"]["include"] += [ + str(pathlib.Path(compiler["image"]["prefix_path"]) / "env/default/packages.yaml") + ] + # add gpu base uenv + if "gpu" in self.base_uenv: + files_config_env["spack"]["include"] += [ + str(pathlib.Path(self.base_uenv["gpu"]["image"]["prefix_path"]) / "env/default/packages.yaml") + ] + files["config"][env] = yaml.dump(files_config_env) return files diff --git a/stackinator/schema.py b/stackinator/schema.py index 0f6379d6..6d8f8f65 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -52,3 +52,5 @@ def py2yaml(data, indent): environments_validator = validator(environments_schema) cache_schema = json.load(open(prefix / "schema/cache.json")) cache_validator = validator(cache_schema) +base_uenv_schema = json.load(open(prefix / "schema/base-uenv.json")) +base_uenv_validator = validator(base_uenv_schema) diff --git a/stackinator/schema/base-uenv.json b/stackinator/schema/base-uenv.json new file mode 100644 index 00000000..cbf6b2f3 --- /dev/null +++ b/stackinator/schema/base-uenv.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Modular Uenv", + "description": "Schema for modular sqfs", + "type": "object", + "required": ["compilers"], + "defs": { + "image": { + "type": "object", + "properties": { + "sha256": { + "type": "string", + "description": "SHA256 hash of the image", + "pattern": "^[0-9a-f]{64}$" + }, + "file": { + "type": "string", + "description": "File path or name of the image" + }, + "name": { + "type": "string", + "description": "uenv spec" + }, + "prefix_path": { + "type": "string", + "description": "mount point" + } + }, + "anyOf": [{ + "required": ["name"] + }, + { + "required": ["sha256"] + } + ], + "additionalProperties": false + } + }, + "properties": { + "compilers": { + "type": "array", + "description": "List of compiler configurations", + "minItems": 1, + "items": { + "type": "object", + "required": ["type", "version", "image"], + "properties": { + "type": { + "type": "string", + "enum": ["gcc", "llvm", "nvhpc"], + "description": "Type of the compiler" + }, + "image": { + "$ref": "#/defs/image" + } + } + } + }, + "root": { + "type": "object", + "properties": { + "image": { + "$ref": "#/defs/image" + } + }, + "description": "The root sqfs image" + }, + "gpu": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "rocm", + "cuda" + ], + "description": "type of the gpu backend" + }, + "image": { + "$ref": "#/defs/image", + "description": "TOOD" + }, + "required": [ + "type", + "image" + ] + }, + "additionalProperties": false + } + } +} diff --git a/stackinator/schema/compilers.json b/stackinator/schema/compilers.json index 1647a5bd..a618b574 100644 --- a/stackinator/schema/compilers.json +++ b/stackinator/schema/compilers.json @@ -3,7 +3,7 @@ "title": "Schema for Spack Stack compilers.yaml recipe file", "type": "object", "additionalProperties": false, - "required": ["bootstrap", "gcc"], + "required": ["gcc"], "defs": { "gcc_version_spec": { "type": "string", diff --git a/stackinator/schema/environments.json b/stackinator/schema/environments.json index 9faefbd6..c11498b6 100644 --- a/stackinator/schema/environments.json +++ b/stackinator/schema/environments.json @@ -6,7 +6,7 @@ "patternProperties": { "\\w[\\w-]*": { "type": "object", - "required": ["compiler", "specs"], + "required": ["specs"], "additionalProperties": false, "properties": { "deprecated": { @@ -17,17 +17,6 @@ "enum": ["when_possible", true, false], "default": true }, - "compiler": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "toolchain": {"type": "string"}, - "spec": {"type": "string"} - } - } - }, "specs": { "type": "array", "items": {"type": "string"} diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index 91ed57d8..cce0c088 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -1,3 +1,4 @@ +{% set spacejoiner = joiner(' ') %} # vim: filetype=make # Copy this file to Make.user and set some variables. @@ -15,6 +16,10 @@ SPACK_HELPER := $(SPACK) --color=never # The Spack installation root. STORE := {{ store }} +# export as env vars (required by bwrap scripts) +export STORE := $(STORE) +export BUILD_ROOT := $(BUILD_ROOT) + # When already building inside a sandbox, use `SANDBOX :=` (empty string) # Without a sandbox, make sure to hide sensitive data such as ~/.ssh through bubblewrap. # Also bind the directories `./tmp -> /tmp` and `./store -> $(STORE)`, so that @@ -22,13 +27,22 @@ STORE := {{ store }} # put the project itself in-memory, or use a flag like --bind /dev/shm/store # $(STORE). Use `bwrap-mutable-root.sh` in case you need to create a new # directory at the root /. +SQFS_MOUNT := $(BUILD_ROOT)/squashfs-mount-wrapper.sh --build-root=$(BUILD_ROOT) --sqfs-images="{% for item in base_uenv %}{{ spacejoiner() }}{{ item.file }}:{{item.prefix_path}}{% endfor %}" {% if no_bwrap %} -SANDBOX := +SANDBOX := $SQFS_MOUNT -- {% else %} -SANDBOX := $(BUILD_ROOT)/bwrap-mutable-root.sh $\ +SANDBOX := $(SQFS_MOUNT) -- $(BUILD_ROOT)/bwrap-mutable-root.sh $\ + --tmpfs ~ $\ + --bind $(BUILD_ROOT)/tmp /tmp $\ + --bind $(BUILD_ROOT)/store $(STORE) $\ +{% for item in base_uenv %} + --bind $(BUILD_ROOT)/tmp/mounts/{{item.prefix_path}} {{item.prefix_path}} $\ +{% endfor %} + +SANDBOX_NO_BASE := $(BUILD_ROOT)/bwrap-mutable-root.sh $\ --tmpfs ~ $\ --bind $(BUILD_ROOT)/tmp /tmp $\ - --bind $(BUILD_ROOT)/store $(STORE) + -- $(BUILD_ROOT)/bwrap-store.sh {% endif %} # Makes sure that make -Orecurse continues to print in color. export SPACK_COLOR := always diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index d0cf8b68..611b02b8 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -51,13 +51,13 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} touch mirror-setup -compilers: mirror-setup - $(SANDBOX) $(MAKE) -C $@ +# compilers: mirror-setup +# $(SANDBOX) $(MAKE) -C $@ -generate-config: compilers +generate-config: mirror-setup $(SANDBOX) $(MAKE) -C $@ -environments: compilers +environments: mirror-setup $(SANDBOX) $(MAKE) -C $@ {% if modules %} @@ -81,9 +81,9 @@ post-install: env-meta # Create a squashfs file from the installed software. store.squashfs: post-install # clean up the __pycache__ paths in the repo - $(SANDBOX) find $(STORE)/repo -type d -name __pycache__ -exec rm -r {} + - $(SANDBOX) chmod -R a+rX $(STORE) - $(SANDBOX) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -e ./compilers/bootstrap find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -all-root -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 + $(SANDBOX_NO_BASE) find $(STORE)/spack_repo -type d -name __pycache__ -exec rm -r {} + + $(SANDBOX_NO_BASE) chmod -R a+rX $(STORE) + $(SANDBOX_NO_BASE) env -u SOURCE_DATE_EPOCH "$$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format='{prefix}' squashfs | head -n1)/bin/mksquashfs" $(STORE) $@ -force-uid nobody -force-gid nobody -all-time $$(date +%s) -no-recovery -noappend -Xcompression-level 3 # Force push all built packages to the build cache cache-force: mirror-setup @@ -110,6 +110,6 @@ build.tar.gz: spack-version Make.user Make.inc Makefile | environments # Clean generate files, does *not* remove installed software. clean: - rm -rf -- $(wildcard */*/spack.lock) $(wildcard */*/.spack-env) $(wildcard */*/Makefile) $(wildcard */*/generated) $(wildcard cache) $(wildcard compilers/*/config.yaml) $(wildcard compilers/*/packages.yaml) $(wildcard compilers/*/compilers.yaml) $(wildcard environments/*/config.yaml) $(wildcard environments/*/packages.yaml) $(wildcard environments/*/compilers.yaml) post-install modules-done env-meta store.squashfs + rm -rf -- $(wildcard */*/spack.lock) $(wildcard */*/.spack-env) $(wildcard */*/Makefile) $(wildcard */*/generated) $(wildcard cache) $(wildcard compilers/*/config.yaml) $(wildcard compilers/*/packages.yaml) $(wildcard compilers/*/compilers.yaml) $(wildcard environments/*/config.yaml) $(wildcard environments/*/packages.yaml) post-install modules-done env-meta store.squashfs include Make.inc diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index f32eb0e5..3b8a97ae 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -41,14 +41,15 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% endif %} {% endfor %} -# Configure dependencies between compilers -gcc/compilers.yaml: bootstrap/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./bootstrap find --format '{prefix}' {{ compilers.gcc.requires }})) +# # Configure dependencies between compilers +# # TODO: remove this and write compilers.yaml from cluster config +# gcc/compilers.yaml: bootstrap/generated/env +# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./bootstrap find --format '{prefix}' {{ compilers.gcc.requires }})) -{% if compilers.llvm %} -llvm/compilers.yaml: gcc/generated/env - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --format '{prefix}' {{ compilers.llvm.requires }})) -{% endif %} +# {% if compilers.llvm %} +# llvm/compilers.yaml: gcc/generated/env +# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --format '{prefix}' {{ compilers.llvm.requires }})) +# {% endif %} include ../Make.inc @@ -57,11 +58,11 @@ include ../Make.inc # specify the order here by conditionally including them, when the dependent exists. ifeq (,$(filter clean,$(MAKECMDGOALS))) -include bootstrap/Makefile +# include bootstrap/Makefile -ifneq (,$(wildcard bootstrap/Makefile)) -include gcc/Makefile -endif +# ifneq (,$(wildcard bootstrap/Makefile)) +# include gcc/Makefile +# endif {% if compilers.llvm %} ifneq (,$(wildcard gcc/Makefile)) diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5f163710..8f3ca671 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -33,7 +33,8 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {{ env }}/generated/view_config: {{ env }}/generated/env {% if config.view %} $(SPACK) env activate --with-view default --sh ./{{ env }} > $(STORE)/env/{{ config.view.name }}/activate.sh - $(BUILD_ROOT)/envvars.py view {% if config.view.extra.add_compilers %}--compilers=./{{ env }}/compilers.yaml {% endif %} --prefix_paths="{{ config.view.extra.prefix_string }}" $(STORE)/env/{{ config.view.name }} $(BUILD_ROOT) + $(BUILD_ROOT)/envvars.py view --prefix_paths="{{ config.view.extra.prefix_string }}" $(STORE)/env/{{ config.view.name }} $(BUILD_ROOT) + $(BUILD_ROOT)/gen_packages_yaml.py --view=$(STORE)/env/{{ config.view.name }} --env=./{{ env }} --lock-file=./{{ env }}/spack.lock {% endif %} touch $@ @@ -42,15 +43,16 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {% for env in environments %}{{ env }}/config.yaml {% endfor %}: | store $(SPACK) config --scope=user add config:install_tree:root:$(STORE) + {# $(SPACK) config --scope=user add config:install_tree:padded_length:128 #} -# Create the compilers.yaml configuration for each environment -{% for env, config in environments.items() %} -{{ env }}_PREFIX = {% for C in config.compiler %} $$($(SPACK_HELPER) -e ../compilers/{{ C.toolchain }} find --format '{prefix}' {{ C.spec }}){% endfor %} +{# # Create the compilers.yaml configuration for each environment #} +{# {% for env, config in environments.items() %} #} +{# {{ env }}_PREFIX = {% for C in config.compiler %} $$($(SPACK_HELPER) -e ../compilers/{{ C.toolchain }} find --format '{prefix}' {{ C.spec }}){% endfor %} #} -{{ env }}/compilers.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $({{ env }}_PREFIX)) +{# {{ env }}/compilers.yaml: #} +{# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $({{ env }}_PREFIX)) #} -{% endfor %} +{# {% endfor %} #} # Configure external system dependencies for each compiler toolchain {% for env, config in environments.items() %} diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index cb6c8528..ae46ecf4 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -3,35 +3,31 @@ include ../Make.user CONFIG_DIR = $(STORE)/config MODULE_DIR = $(BUILD_ROOT)/modules -# These will be the prefixes of the GCCs, LLVMs and NVHPCs in the respective environments. -ALL_COMPILER_PREFIXES ={% for compiler in all_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} +# COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} -COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} +all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml - -all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/compilers.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml $(MODULE_DIR)/compilers.yaml - -# Generate the upstream configuration that will be provided by the mounted image -$(CONFIG_DIR)/compilers.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(COMPILER_PREFIXES)) +# # Generate the upstream configuration that will be provided by the mounted image +# $(CONFIG_DIR)/compilers.yaml: +# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(COMPILER_PREFIXES)) $(CONFIG_DIR)/upstreams.yaml: $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) # Copy the cluster-specific packages.yaml file to the configuration. # requires compilers.yaml to ensure that the path $(CONFIG_DIR) has been created. -$(CONFIG_DIR)/packages.yaml: $(CONFIG_DIR)/compilers.yaml - install -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml +$(CONFIG_DIR)/packages.yaml: + install -D -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml -$(CONFIG_DIR)/repos.yaml: $(CONFIG_DIR)/compilers.yaml - install -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml +$(CONFIG_DIR)/repos.yaml: + install -D -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml # Generate a configuration used to generate the module files # The configuration in CONFIG_DIR can't be used for this purpose, because a compilers.yaml # that includes the bootstrap compiler is required to build the modules. -$(MODULE_DIR)/compilers.yaml: - $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(ALL_COMPILER_PREFIXES)) +# $(MODULE_DIR)/compilers.yaml: +# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(ALL_COMPILER_PREFIXES)) $(MODULE_DIR)/upstreams.yaml: $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) diff --git a/stackinator/templates/compilers.bootstrap.spack.yaml b/stackinator/templates/compilers.bootstrap.spack.yaml deleted file mode 100644 index 88c0b541..00000000 --- a/stackinator/templates/compilers.bootstrap.spack.yaml +++ /dev/null @@ -1,26 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: true - reuse: false - packages: - gcc: - variants: [build_type=Release ~bootstrap +strip] - mpc: - variants: [libs=static] - gmp: - variants: [libs=static] - mpfr: - variants: [libs=static] - zstd: - variants: [libs=static] - zlib: - variants: [~shared] - diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml index cf186e45..ac9d89ad 100644 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ b/stackinator/templates/compilers.gcc.spack.yaml @@ -2,7 +2,6 @@ spack: include: - packages.yaml - config.yaml - - compilers.yaml specs: {% for spec in config.specs %} - {{ spec }} diff --git a/stackinator/templates/compilers.llvm.spack.yaml b/stackinator/templates/compilers.llvm.spack.yaml index c7882ed0..e567b226 100644 --- a/stackinator/templates/compilers.llvm.spack.yaml +++ b/stackinator/templates/compilers.llvm.spack.yaml @@ -2,7 +2,6 @@ spack: include: - packages.yaml - config.yaml - - compilers.yaml specs: {% for spec in config.specs %} - {{ spec }} diff --git a/stackinator/templates/environments.spack.yaml b/stackinator/templates/environments.spack.yaml index 2477eea6..27287215 100644 --- a/stackinator/templates/environments.spack.yaml +++ b/stackinator/templates/environments.spack.yaml @@ -3,7 +3,6 @@ spack: {% if config.packages %} - packages.yaml {% endif %} - - compilers.yaml - config.yaml config: deprecated: {{ config.deprecated }} @@ -14,23 +13,25 @@ spack: {% for spec in config.specs %} - '{{ spec }}' {% endfor %} +{% if config.toolchain_constraints or config.variants or config.mpi.spec %} packages: + {% if config.toolchain_constraints or config.variants %} all: -{% set separator = joiner(', ') %} - compiler: [{% for c in config.compiler %}{{ separator() }}'{{ c.spec }}'{% endfor %}] + {% endif %} {% if config.toolchain_constraints %} require: {% set separator = joiner(', ') %} - one_of: [{% for c in config.toolchain_constraints %}{{ separator() }}'{{ c }}'{% endfor %}] {% endif %} - {% if config.variants %} +{% if config.variants %} {% set separator = joiner(', ') %} variants: [{% for v in config.variants %}{{ separator() }}'{{ v }}'{% endfor %}] - {% endif %} - {% if config.mpi.spec %} +{% endif %} +{% if config.mpi.spec %} mpi: require: '{{ config.mpi.spec }}' - {% endif %} +{% endif %} +{% endif %} {% if config.view %} view: default: diff --git a/stackinator/templates/stack-debug.sh b/stackinator/templates/stack-debug.sh index b1efc050..ee442cae 100644 --- a/stackinator/templates/stack-debug.sh +++ b/stackinator/templates/stack-debug.sh @@ -1 +1,14 @@ -env --ignore-environment PATH=/usr/bin:/bin:{{ build_path }}/spack/bin HOME=$HOME BUILD_ROOT={{ build_path }} STORE={{ mount_path }} SPACK_SYSTEM_CONFIG_PATH={{ build_path }}/config SPACK_USER_CACHE_PATH={{ build_path }}/cache SPACK=spack SPACK_COLOR=always SPACK_USER_CONFIG_PATH={% if spack_version>="0.23" %}~{% else %}/dev/null{% endif %} LC_ALL=en_US.UTF-8 TZ=UTC SOURCE_DATE_EPOCH=315576060 {% if use_bwrap %} {{ build_path }}/bwrap-mutable-root.sh --tmpfs ~ --bind {{ build_path }}/tmp /tmp --bind {{ build_path }}/store {{ mount_path }} {% endif %} bash -noprofile -l +#!/bin/bash + +set -eu + +env --ignore-environment \ + PATH=/usr/bin:/bin:{{ build_path }}/spack/bin \ + HOME=$HOME BUILD_ROOT={{ build_path }} \ + STORE={{ mount_path }} SPACK_SYSTEM_CONFIG_PATH={{ build_path }}/config \ + SPACK_USER_CACHE_PATH={{ build_path }}/cache \ + SPACK=spack SPACK_COLOR=always \ + SPACK_USER_CONFIG_PATH={% if spack_version>="0.23" %}~{% else %}/dev/null{% endif %} \ + LC_ALL=en_US.UTF-8 TZ=UTC SOURCE_DATE_EPOCH=315576060 \ + {% if use_bwrap %} {{ build_path }}/bwrap-mutable-root.sh --tmpfs ~ --bind {{ build_path }}/tmp /tmp -- {{ build_path }}/bwrap-store.sh {% endif %} \ + bash -noprofile -l From a51de61e794568f7581cad6608f9f6eefbcf86ac Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Fri, 6 Jun 2025 00:56:15 +0200 Subject: [PATCH 4/8] remove unused Makefiler.compilers --- stackinator/templates/Makefile.compilers | 74 ------------------------ 1 file changed, 74 deletions(-) delete mode 100644 stackinator/templates/Makefile.compilers diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers deleted file mode 100644 index 3b8a97ae..00000000 --- a/stackinator/templates/Makefile.compilers +++ /dev/null @@ -1,74 +0,0 @@ -{% set pipejoiner = joiner('|') %} --include ../Make.user - -MAKEFLAGS += --output-sync=recurse - -.PHONY: all .locks .packages.yaml - -all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfor %} - - -# Ensure that spack.lock files are never removed as intermediate files... -.locks:{% for compiler in compilers %} {{ compiler }}/spack.lock{% endfor %} - - -# Ensure that package yaml files are never removed as intermediate files... -.packages.yaml:{% for compiler in compilers %} {{ compiler }}/packages.yaml{% endfor %} - - -{% for compiler, config in compilers.items() %} -{{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ - $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ - | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ - | cut -d ';' -f2) -{% endif %} - touch $@ - -{% endfor %} - -# Configure the install location. -{% for compiler in compilers %}{{ compiler }}/config.yaml {% endfor %}: | store - $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - -# Configure external system dependencies for each compiler toolchain -{% for compiler, config in compilers.items() %} -{% if config.packages and config.packages.external %} -{{ compiler }}/packages.yaml: - $(SPACK) external find --scope=user {% for package in config.packages.external %} {{package}}{% endfor %} - - -{% endif %} -{% endfor %} -# # Configure dependencies between compilers -# # TODO: remove this and write compilers.yaml from cluster config -# gcc/compilers.yaml: bootstrap/generated/env -# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./bootstrap find --format '{prefix}' {{ compilers.gcc.requires }})) - -# {% if compilers.llvm %} -# llvm/compilers.yaml: gcc/generated/env -# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $$($(SPACK_HELPER) -e ./gcc find --format '{prefix}' {{ compilers.llvm.requires }})) -# {% endif %} - - -include ../Make.inc - -# GNU Make isn't very smart about dependencies across included Makefiles, so we -# specify the order here by conditionally including them, when the dependent exists. -ifeq (,$(filter clean,$(MAKECMDGOALS))) - -# include bootstrap/Makefile - -# ifneq (,$(wildcard bootstrap/Makefile)) -# include gcc/Makefile -# endif - -{% if compilers.llvm %} -ifneq (,$(wildcard gcc/Makefile)) -include llvm/Makefile -endif -{% endif %} - - -endif From 0f932c706526bf62697eb2280d3daad1f23d1763 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Tue, 17 Jun 2025 16:05:01 +0200 Subject: [PATCH 5/8] format --- stackinator/recipe.py | 51 ++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 48fb4a5a..d96358b3 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -89,9 +89,7 @@ def __init__(self, args): environments_path = self.path / "environments.yaml" self._logger.debug(f"opening {environments_path}") if not environments_path.is_file(): - raise FileNotFoundError( - f"The recipe path '{environments_path}' does not contain environments.yaml" - ) + raise FileNotFoundError(f"The recipe path '{environments_path}' does not contain environments.yaml") with environments_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) @@ -121,9 +119,7 @@ def __init__(self, args): modules_path = self.path / "modules.yaml" self._logger.debug(f"opening {modules_path}") if not modules_path.is_file(): - modules_path = ( - pathlib.Path(args.build) / "spack/etc/spack/defaults/modules.yaml" - ) + modules_path = pathlib.Path(args.build) / "spack/etc/spack/defaults/modules.yaml" self._logger.debug(f"no modules.yaml provided - using the {modules_path}") self.modules = modules_path @@ -225,13 +221,9 @@ def mirror(self, configuration): if file is not None: mirror_config_path = pathlib.Path(file) if not mirror_config_path.is_file(): - raise FileNotFoundError( - f"The cache configuration '{file}' is not a file" - ) + raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - self._mirror = cache.configuration_from_file( - mirror_config_path, pathlib.Path(mount) - ) + self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) @property def config(self): @@ -241,9 +233,7 @@ def config(self): def config(self, config_path): self._logger.debug(f"opening {config_path}") if not config_path.is_file(): - raise FileNotFoundError( - f"The recipe path '{config_path}' does not contain config.yaml" - ) + raise FileNotFoundError(f"The recipe path '{config_path}' does not contain config.yaml") with config_path.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) @@ -274,9 +264,7 @@ def environment_view_meta(self): def modules_yaml(self): with self.modules.open() as fid: raw = yaml.load(fid, Loader=yaml.Loader) - raw["modules"]["default"]["roots"]["tcl"] = ( - pathlib.Path(self.mount) / "modules" - ).as_posix() + raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix() return yaml.dump(raw) # creates the self.environments field that describes the full specifications @@ -357,9 +345,7 @@ def generate_environment_specs(self, raw): env_name_map[name] = [] for view, vc in config["views"].items(): if view in env_names: - raise Exception( - f"An environment view with the name '{view}' already exists." - ) + raise Exception(f"An environment view with the name '{view}' already exists.") # set some default values: # vc["link"] = "roots" # vc["uenv"]["add_compilers"] = True @@ -371,10 +357,7 @@ def generate_environment_specs(self, raw): vc["uenv"].setdefault("add_compilers", True) vc["uenv"].setdefault("prefix_paths", {}) prefix_string = ",".join( - [ - f"{name}={':'.join(paths)}" - for name, paths in vc["uenv"]["prefix_paths"].items() - ] + [f"{name}={':'.join(paths)}" for name, paths in vc["uenv"]["prefix_paths"].items()] ) vc["uenv"]["prefix_string"] = prefix_string # save a copy of the view configuration @@ -431,9 +414,7 @@ def system_config_path(self, path): system_path = pathlib.Path.cwd() / system_path if not system_path.is_dir(): - raise FileNotFoundError( - f"The system configuration path '{system_path}' does not exist" - ) + raise FileNotFoundError(f"The system configuration path '{system_path}' does not exist") self._system_path = system_path @@ -463,25 +444,17 @@ def environment_files(self): files["config"] = {} for env, config in self.environments.items(): spack_yaml_template = jenv.get_template("environments.spack.yaml") - files["config"][env] = spack_yaml_template.render( - config=config, name=env, store=self.mount - ) + files["config"][env] = spack_yaml_template.render(config=config, name=env, store=self.mount) files_config_env = yaml.safe_load(files["config"][env]) # add base uenv upstream for compiler in self.base_uenv["compilers"]: files_config_env["spack"]["include"] += [ - str( - pathlib.Path(compiler["image"]["prefix_path"]) - / "env/default/packages.yaml" - ) + str(pathlib.Path(compiler["image"]["prefix_path"]) / "env/default/packages.yaml") ] # add gpu base uenv if "gpu" in self.base_uenv: files_config_env["spack"]["include"] += [ - str( - pathlib.Path(self.base_uenv["gpu"]["image"]["prefix_path"]) - / "env/default/packages.yaml" - ) + str(pathlib.Path(self.base_uenv["gpu"]["image"]["prefix_path"]) / "env/default/packages.yaml") ] files["config"][env] = yaml.dump(files_config_env) return files From bc8bf632b4add04940a89d7ccef81e1255d82b2d Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Wed, 18 Jun 2025 10:14:04 +0200 Subject: [PATCH 6/8] cleanup --- stackinator/recipe.py | 14 -------------- stackinator/templates/Makefile | 3 --- stackinator/templates/Makefile.environments | 10 ---------- stackinator/templates/Makefile.generate-config | 3 --- 4 files changed, 30 deletions(-) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index d96358b3..70517179 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -315,20 +315,6 @@ def generate_environment_specs(self, raw): # TODO: Create a custom exception type raise Exception(f"Unsupported mpi: {mpi_impl}") - # set constraints that ensure the the main compiler is always used to build packages - # that do not explicitly request a compiler. - # for name, config in environments.items(): - # compilers = config["compiler"] - # if len(compilers) == 1: - # config["toolchain_constraints"] = [] - # continue - # requires = [f"%{compilers[0]['spec']}"] - # for spec in config["specs"]: - # if "%" in spec: - # requires.append(spec) - - # config["toolchain_constraints"] = requires - # An awkward hack to work around spack not supporting creating activation # scripts for each file system view in an environment: it only generates them # for the "default" view. diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 611b02b8..46c6dfbc 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -51,9 +51,6 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} touch mirror-setup -# compilers: mirror-setup -# $(SANDBOX) $(MAKE) -C $@ - generate-config: mirror-setup $(SANDBOX) $(MAKE) -C $@ diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 8f3ca671..13bf577b 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -43,16 +43,6 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {% for env in environments %}{{ env }}/config.yaml {% endfor %}: | store $(SPACK) config --scope=user add config:install_tree:root:$(STORE) - {# $(SPACK) config --scope=user add config:install_tree:padded_length:128 #} - -{# # Create the compilers.yaml configuration for each environment #} -{# {% for env, config in environments.items() %} #} -{# {{ env }}_PREFIX = {% for C in config.compiler %} $$($(SPACK_HELPER) -e ../compilers/{{ C.toolchain }} find --format '{prefix}' {{ C.spec }}){% endfor %} #} - -{# {{ env }}/compilers.yaml: #} -{# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $({{ env }}_PREFIX)) #} - -{# {% endfor %} #} # Configure external system dependencies for each compiler toolchain {% for env, config in environments.items() %} diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index ae46ecf4..6a0f3430 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -8,9 +8,6 @@ MODULE_DIR = $(BUILD_ROOT)/modules all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml -# # Generate the upstream configuration that will be provided by the mounted image -# $(CONFIG_DIR)/compilers.yaml: -# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(COMPILER_PREFIXES)) $(CONFIG_DIR)/upstreams.yaml: $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) From 9f9c85f8fd92559037539cc1604fa8df4c2112a9 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Wed, 18 Jun 2025 10:17:34 +0200 Subject: [PATCH 7/8] cleanup --- stackinator/templates/Makefile.generate-config | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/stackinator/templates/Makefile.generate-config b/stackinator/templates/Makefile.generate-config index 6a0f3430..c726903e 100644 --- a/stackinator/templates/Makefile.generate-config +++ b/stackinator/templates/Makefile.generate-config @@ -6,7 +6,7 @@ MODULE_DIR = $(BUILD_ROOT)/modules # COMPILER_PREFIXES ={% for compiler in release_compilers %} $$($(SPACK_HELPER) -e ../compilers/{{ compiler }} find --format='{prefix}' gcc llvm nvhpc){% endfor %} -all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml +all: $(CONFIG_DIR)/upstreams.yaml $(CONFIG_DIR)/packages.yaml $(CONFIG_DIR)/repos.yaml $(MODULE_DIR)/upstreams.yaml $(CONFIG_DIR)/upstreams.yaml: @@ -14,18 +14,12 @@ $(CONFIG_DIR)/upstreams.yaml: # Copy the cluster-specific packages.yaml file to the configuration. # requires compilers.yaml to ensure that the path $(CONFIG_DIR) has been created. -$(CONFIG_DIR)/packages.yaml: +$(CONFIG_DIR)/packages.yaml: install -D -m 644 $(BUILD_ROOT)/config/packages.yaml $(CONFIG_DIR)/packages.yaml -$(CONFIG_DIR)/repos.yaml: +$(CONFIG_DIR)/repos.yaml: install -D -m 644 $(BUILD_ROOT)/config/repos.yaml $(CONFIG_DIR)/repos.yaml -# Generate a configuration used to generate the module files -# The configuration in CONFIG_DIR can't be used for this purpose, because a compilers.yaml -# that includes the bootstrap compiler is required to build the modules. -# $(MODULE_DIR)/compilers.yaml: -# $(SPACK) compiler find --scope=user $(call compiler_bin_dirs, $(ALL_COMPILER_PREFIXES)) - $(MODULE_DIR)/upstreams.yaml: $(SPACK) config --scope=user add upstreams:system:install_tree:$(STORE) From 146b453692902af85c9afb7ff9e20c395939b196 Mon Sep 17 00:00:00 2001 From: Simon Pintarelli Date: Wed, 18 Jun 2025 16:38:29 +0200 Subject: [PATCH 8/8] delete unused templates --- .../templates/compilers.gcc.spack.yaml | 25 ------------------- .../templates/compilers.llvm.spack.yaml | 13 ---------- 2 files changed, 38 deletions(-) delete mode 100644 stackinator/templates/compilers.gcc.spack.yaml delete mode 100644 stackinator/templates/compilers.llvm.spack.yaml diff --git a/stackinator/templates/compilers.gcc.spack.yaml b/stackinator/templates/compilers.gcc.spack.yaml deleted file mode 100644 index ada22cb1..00000000 --- a/stackinator/templates/compilers.gcc.spack.yaml +++ /dev/null @@ -1,25 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false - packages: - gcc: - variants: [build_type=Release +bootstrap +strip] - mpc: - variants: [libs=static] - gmp: - variants: [libs=static] - mpfr: - variants: [libs=static] - zstd: - variants: [libs=static] - zlib: - variants: [~shared] diff --git a/stackinator/templates/compilers.llvm.spack.yaml b/stackinator/templates/compilers.llvm.spack.yaml deleted file mode 100644 index e567b226..00000000 --- a/stackinator/templates/compilers.llvm.spack.yaml +++ /dev/null @@ -1,13 +0,0 @@ -spack: - include: - - packages.yaml - - config.yaml - specs: -{% for spec in config.specs %} - - {{ spec }} -{% endfor %} - view: false - concretizer: - unify: when_possible - reuse: false -