diff --git a/backend/src/hatchling/build.py b/backend/src/hatchling/build.py index 5a1da4f89..eb1a65422 100644 --- a/backend/src/hatchling/build.py +++ b/backend/src/hatchling/build.py @@ -3,6 +3,8 @@ import os from typing import Any +from hatchling.builders.variant_constants import VARIANT_DIST_INFO_FILENAME + __all__ = [ 'build_editable', 'build_sdist', @@ -116,6 +118,10 @@ def prepare_metadata_for_build_wheel( with open(os.path.join(directory, 'METADATA'), 'w', encoding='utf-8') as f: f.write(builder.config.core_metadata_constructor(builder.metadata)) + if builder.metadata.variant_hash is not None: + with open(os.path.join(directory, VARIANT_DIST_INFO_FILENAME), 'w', encoding='utf-8') as f: + f.write(builder.config.variants_json_constructor(builder.metadata.variant_config)) + return os.path.basename(directory) def prepare_metadata_for_build_editable( diff --git a/backend/src/hatchling/builders/variant_constants.py b/backend/src/hatchling/builders/variant_constants.py new file mode 100644 index 000000000..fd572e2d3 --- /dev/null +++ b/backend/src/hatchling/builders/variant_constants.py @@ -0,0 +1,143 @@ +# This file is copied from variantlib/variantlib/constants.py +# Do not edit this file directly, instead edit variantlib/variantlib/constants.py + +from __future__ import annotations + +import re +from typing import Literal, TypedDict + +VARIANT_HASH_LEN = 8 +CONFIG_FILENAME = "variants.toml" +VARIANT_DIST_INFO_FILENAME = "variant.json" + +# Common variant info keys (used in pyproject.toml and variants.json) +VARIANT_INFO_DEFAULT_PRIO_KEY: Literal["default-priorities"] = "default-priorities" +VARIANT_INFO_FEATURE_KEY: Literal["feature"] = "feature" +VARIANT_INFO_NAMESPACE_KEY: Literal["namespace"] = "namespace" +VARIANT_INFO_PROPERTY_KEY: Literal["property"] = "property" +VARIANT_INFO_PROVIDER_DATA_KEY: Literal["providers"] = "providers" +VARIANT_INFO_PROVIDER_ENABLE_IF_KEY: Literal["enable-if"] = "enable-if" +VARIANT_INFO_PROVIDER_OPTIONAL_KEY: Literal["optional"] = "optional" +VARIANT_INFO_PROVIDER_PLUGIN_API_KEY: Literal["plugin-api"] = "plugin-api" +VARIANT_INFO_PROVIDER_REQUIRES_KEY: Literal["requires"] = "requires" + +PYPROJECT_TOML_TOP_KEY = "variant" + +VARIANTS_JSON_SCHEMA_KEY: Literal["$schema"] = "$schema" +VARIANTS_JSON_SCHEMA_URL = "https://variants-schema.wheelnext.dev/" +VARIANTS_JSON_VARIANT_DATA_KEY: Literal["variants"] = "variants" + +VALIDATION_VARIANT_HASH_REGEX = re.compile(rf"[0-9a-f]{{{VARIANT_HASH_LEN}}}") + +VALIDATION_NAMESPACE_REGEX = re.compile(r"[a-z0-9_]+") +VALIDATION_FEATURE_NAME_REGEX = re.compile(r"[a-z0-9_]+") + +# For `Property value` there is two regexes: +# 1. `VALIDATION_VALUE_VSPEC_REGEX` - if `packaging.specifiers.SpecifierSet` is used +# Note: for clarity - only "full version" are allowed +# i.e. so no "a|b|alpha|beta|rc|post|etc." versions +VALIDATION_VALUE_VSPEC_REGEX = re.compile(r"[0-9_.,!>~<=]+") +# 2. `VALIDATION_VALUE_STR_REGEX` - if string matching is used +VALIDATION_VALUE_STR_REGEX = re.compile(r"[a-z0-9_.]+") +VALIDATION_VALUE_REGEX = re.compile( + rf"{VALIDATION_VALUE_VSPEC_REGEX.pattern}|{VALIDATION_VALUE_STR_REGEX.pattern}" +) + +VALIDATION_FEATURE_REGEX = re.compile( + rf""" + (?P{VALIDATION_NAMESPACE_REGEX.pattern}) + \s* :: \s* + (?P{VALIDATION_FEATURE_NAME_REGEX.pattern}) +""", + re.VERBOSE, +) + +VALIDATION_PROPERTY_REGEX = re.compile( + rf""" + (?P{VALIDATION_NAMESPACE_REGEX.pattern}) + \s* :: \s* + (?P{VALIDATION_FEATURE_NAME_REGEX.pattern}) + \s* :: \s* + (?P{VALIDATION_VALUE_REGEX.pattern}) +""", + re.VERBOSE, +) + +VALIDATION_PROVIDER_ENABLE_IF_REGEX = re.compile(r"[\S ]+") +VALIDATION_PROVIDER_PLUGIN_API_REGEX = re.compile( + r""" + (?P [\w.]+) + (?: \s* : \s* + (?P [\w.]+) + )? + """, + re.VERBOSE, +) +VALIDATION_PROVIDER_REQUIRES_REGEX = re.compile(r"[\S ]+") + + +# VALIDATION_PYTHON_PACKAGE_NAME_REGEX = re.compile(r"[^\s-]+?") +# Per PEP 508: https://peps.python.org/pep-0508/#names +VALIDATION_PYTHON_PACKAGE_NAME_REGEX = re.compile( + r"[A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9]", re.IGNORECASE +) +VALIDATION_WHEEL_NAME_REGEX = re.compile( + r"(?P " # group (without variant) + r" (?P " # "namever" group contains - + r" (?P[^\s-]+?) " # + r" - (?P[^\s-]*?) " # "-" + r" ) " # close "namever" group + r" ( - (?P\d[^-]*?) )? " # optional "-" + r" - (?P[^\s-]+?) " # "-" tag + r" - (?P[^\s-]+?) " # "-" tag + r" - (?P[^\s-]+?) " # "-" tag + r") " # end of group + r"( - (?P " # optional + rf" [0-9a-f]{{{VARIANT_HASH_LEN}}} " + r" ) " + r")? " + r"\.whl " # ".whl" suffix + r" ", + re.VERBOSE, +) + + +# ======================== Json TypedDict for the JSON format ======================== # + +# NOTE: Unfortunately, it is not possible as of today to use variables in the definition +# of TypedDict. Similarly also impossible to use the normal "class format" if a +# key uses the characted `-`. +# +# For all these reasons and easier future maintenance - these classes have been +# added to this file instead of a more "format definition" file. + + +class PriorityJsonDict(TypedDict, total=False): + namespace: list[str] + feature: dict[str, list[str]] + property: dict[str, dict[str, list[str]]] + + +ProviderPluginJsonDict = TypedDict( + "ProviderPluginJsonDict", + { + "plugin-api": str, + "requires": list[str], + "enable-if": str, + }, + total=False, +) + +VariantInfoJsonDict = dict[str, dict[str, list[str]]] + + +VariantsJsonDict = TypedDict( + "VariantsJsonDict", + { + "$schema": str, + "default-priorities": PriorityJsonDict, + "providers": dict[str, ProviderPluginJsonDict], + "variants": dict[str, VariantInfoJsonDict], + }, + total=False, +) \ No newline at end of file diff --git a/backend/src/hatchling/builders/wheel.py b/backend/src/hatchling/builders/wheel.py index e6bc55262..86e49e55b 100644 --- a/backend/src/hatchling/builders/wheel.py +++ b/backend/src/hatchling/builders/wheel.py @@ -2,15 +2,22 @@ import csv import hashlib +import json import os import stat import sys import tempfile import zipfile +from collections import defaultdict from functools import cached_property from io import StringIO from typing import TYPE_CHECKING, Any, Callable, Iterable, NamedTuple, Sequence, Tuple, cast +if TYPE_CHECKING: + from hatchling.bridge.app import Application + from hatchling.metadata.core import ProjectMetadata + from hatchling.plugin.manager import PluginManagerBound + from hatchling.__about__ import __version__ from hatchling.builders.config import BuilderConfig from hatchling.builders.constants import EDITABLES_REQUIREMENT @@ -26,6 +33,23 @@ replace_file, set_zip_info_mode, ) +from hatchling.builders.variant_constants import ( + VALIDATION_PROPERTY_REGEX, + VARIANT_DIST_INFO_FILENAME, + VARIANT_INFO_DEFAULT_PRIO_KEY, + VARIANT_INFO_FEATURE_KEY, + VARIANT_INFO_NAMESPACE_KEY, + VARIANT_INFO_PROPERTY_KEY, + VARIANT_INFO_PROVIDER_DATA_KEY, + VARIANT_INFO_PROVIDER_ENABLE_IF_KEY, + VARIANT_INFO_PROVIDER_OPTIONAL_KEY, + VARIANT_INFO_PROVIDER_PLUGIN_API_KEY, + VARIANT_INFO_PROVIDER_REQUIRES_KEY, + VARIANTS_JSON_SCHEMA_KEY, + VARIANTS_JSON_SCHEMA_URL, + VARIANTS_JSON_VARIANT_DATA_KEY, +) +from hatchling.metadata.core import VariantConfig from hatchling.metadata.spec import DEFAULT_METADATA_VERSION, get_core_metadata_constructors if TYPE_CHECKING: @@ -188,6 +212,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.__core_metadata_constructor: Callable[..., str] | None = None + self.__variants_json_constructor: Callable[..., str] | None = None self.__shared_data: dict[str, str] | None = None self.__shared_scripts: dict[str, str] | None = None self.__extra_metadata: dict[str, str] | None = None @@ -282,6 +307,78 @@ def core_metadata_constructor(self) -> Callable[..., str]: return self.__core_metadata_constructor + @property + def variants_json_constructor(self) -> Callable[..., str]: + if self.__variants_json_constructor is None: + def constructor(variant_config: VariantConfig) -> str: + data = { + VARIANTS_JSON_SCHEMA_KEY: VARIANTS_JSON_SCHEMA_URL, + VARIANT_INFO_DEFAULT_PRIO_KEY: {}, + VARIANT_INFO_PROVIDER_DATA_KEY: {}, + VARIANTS_JSON_VARIANT_DATA_KEY: {} + } + + # ==================== VARIANT_INFO_DEFAULT_PRIO_KEY ==================== # + + if (ns_prio := variant_config.default_priorities["namespace"]): + data[VARIANT_INFO_DEFAULT_PRIO_KEY][VARIANT_INFO_NAMESPACE_KEY] = ns_prio + + if (feat_prio := variant_config.default_priorities["feature"]): + data[VARIANT_INFO_DEFAULT_PRIO_KEY][VARIANT_INFO_FEATURE_KEY] = feat_prio + + if (prop_prio := variant_config.default_priorities["property"]): + data[VARIANT_INFO_DEFAULT_PRIO_KEY][VARIANT_INFO_PROPERTY_KEY] = prop_prio + + if not data[VARIANT_INFO_DEFAULT_PRIO_KEY]: + # If no default priorities are set, remove the key + del data[VARIANT_INFO_DEFAULT_PRIO_KEY] + + # ==================== VARIANT_INFO_PROVIDER_DATA_KEY ==================== # + + variant_providers = defaultdict(dict) + for ns, provider_cfg in variant_config.providers.items(): + variant_providers[ns][VARIANT_INFO_PROVIDER_REQUIRES_KEY] = provider_cfg.requires + + if provider_cfg.enable_if is not None: + variant_providers[ns][VARIANT_INFO_PROVIDER_ENABLE_IF_KEY] = provider_cfg.enable_if + + if provider_cfg.optional: + variant_providers[ns][VARIANT_INFO_PROVIDER_OPTIONAL_KEY] = True + + if provider_cfg.plugin_api is not None: + variant_providers[ns][VARIANT_INFO_PROVIDER_PLUGIN_API_KEY] = provider_cfg.plugin_api + + data[VARIANT_INFO_PROVIDER_DATA_KEY] = variant_providers + + # ==================== VARIANTS_JSON_VARIANT_DATA_KEY ==================== # + + variant_data = defaultdict(lambda: defaultdict(set)) + for vprop_str in (variant_config.properties or []): + match = VALIDATION_PROPERTY_REGEX.match(vprop_str) + if not match: + raise ValueError( + f"Invalid variant property '{vprop_str}' in variant {variant_config.variant_hash}" + ) + namespace = match.group('namespace') + feature = match.group('feature') + value = match.group('value') + variant_data[namespace][feature].add(value) + data[VARIANTS_JSON_VARIANT_DATA_KEY][variant_config.vhash] = variant_data + + def preprocess(data): + """Preprocess the data to ensure it is JSON serializable.""" + if isinstance(data, (defaultdict, dict)): + return {k: preprocess(v) for k, v in data.items()} + if isinstance(data, set): + return list(data) + return data + + return json.dumps( + preprocess(data), indent=4, sort_keys=True, ensure_ascii=False + ) + self.__variants_json_constructor = constructor + return self.__variants_json_constructor + @property def shared_data(self) -> dict[str, str]: if self.__shared_data is None: @@ -449,6 +546,32 @@ class WheelBuilder(BuilderInterface): PLUGIN_NAME = 'wheel' + def __init__( + self, + root: str, + plugin_manager: PluginManagerBound | None = None, + config: dict[str, Any] | None = None, + metadata: ProjectMetadata | None = None, + app: Application | None = None, + variant_props: list[str] | None = None, + variant_label: str | None = None, + ): + metadata.variant_config = VariantConfig.from_dict( + data=metadata.variant_config_data, + vprops=variant_props, + variant_label=variant_label, + ) + metadata.variant_config.validate() + metadata.variant_hash = metadata.variant_config.vhash + + super().__init__( + root=root, + plugin_manager=plugin_manager, + config=config, + metadata=metadata, + app=app, + ) + def get_version_api(self) -> dict[str, Callable]: return {'standard': self.build_standard, 'editable': self.build_editable} @@ -483,7 +606,11 @@ def build_standard(self, directory: str, **build_data: Any) -> str: records.write((f'{archive.metadata_directory}/RECORD', '', '')) archive.write_metadata('RECORD', records.construct()) - target = os.path.join(directory, f"{self.artifact_project_id}-{build_data['tag']}.whl") + if self.metadata.variant_hash is not None: + wheel_name = f"{self.artifact_project_id}-{build_data['tag']}-{self.metadata.variant_hash}.whl" + else: + wheel_name = f"{self.artifact_project_id}-{build_data['tag']}.whl" + target = os.path.join(directory, wheel_name) replace_file(archive.path, target) normalize_artifact_permissions(target) @@ -710,6 +837,12 @@ def write_project_metadata( 'METADATA', self.config.core_metadata_constructor(self.metadata, extra_dependencies=extra_dependencies) ) records.write(record) + if self.metadata.variant_hash is not None: + record = archive.write_metadata( + VARIANT_DIST_INFO_FILENAME, + self.config.variants_json_constructor(self.metadata.variant_config), + ) + records.write(record) def add_licenses(self, archive: WheelArchive, records: RecordFile) -> None: for relative_path in self.metadata.core.license_files: diff --git a/backend/src/hatchling/cli/build/__init__.py b/backend/src/hatchling/cli/build/__init__.py index 1e4f1d35d..3eedd67f9 100644 --- a/backend/src/hatchling/cli/build/__init__.py +++ b/backend/src/hatchling/cli/build/__init__.py @@ -15,6 +15,9 @@ def build_impl( clean_hooks_after: bool, clean_only: bool, show_dynamic_deps: bool, + variant_props: list[str] | None = None, + variant_null: bool = False, + variant_label: str | None, ) -> None: import os @@ -72,7 +75,21 @@ def build_impl( if not (clean_only or show_dynamic_deps) and len(target_data) > 1: app.display_mini_header(target_name) - builder = builder_class(root, plugin_manager=plugin_manager, metadata=metadata, app=app.get_safe_application()) + variant_build_kwargs = {} + if f'{builder_class.__module__}.{builder_class.__name__}' == 'hatchling.builders.wheel.WheelBuilder': + variant_build_kwargs = { + "variant_props": variant_props if not variant_null else [], + "variant_label": variant_label, + } + + builder = builder_class( + root, + plugin_manager=plugin_manager, + metadata=metadata, + app=app.get_safe_application(), + **variant_build_kwargs, + ) + if show_dynamic_deps: for dependency in builder.config.dynamic_dependencies: dynamic_dependencies[dependency] = None @@ -116,4 +133,28 @@ def build_command(subparsers: argparse._SubParsersAction, defaults: Any) -> None parser.add_argument('--clean-only', dest='clean_only', action='store_true') parser.add_argument('--show-dynamic-deps', dest='show_dynamic_deps', action='store_true') parser.add_argument('--app', dest='called_by_app', action='store_true', help=argparse.SUPPRESS) + + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument( + '-p', + '--variant-property', + dest='variant_props', + type=str, + action='extend', + nargs='+', + help=('Variant Properties to add to the Wheel Variant, can be repeated as many times as needed'), + default=None, + ) + group.add_argument( + '--null-variant', + dest='variant_null', + action='store_true', + help='Make the variant a `null variant` - no variant property.', + ) + parser.add_argument( + '--variant-label', + dest='variant_label', + help='Use a custom variant label (the default is variant hash)', + ) + parser.set_defaults(func=build_impl) diff --git a/backend/src/hatchling/metadata/core.py b/backend/src/hatchling/metadata/core.py index 88ed55775..c823ea63d 100644 --- a/backend/src/hatchling/metadata/core.py +++ b/backend/src/hatchling/metadata/core.py @@ -1,9 +1,11 @@ from __future__ import annotations +import hashlib import os import sys from contextlib import suppress from copy import deepcopy +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Generic, cast from hatchling.metadata.utils import ( @@ -31,11 +33,101 @@ import tomli as tomllib +VARIANT_HASH_LEN = 8 + + def load_toml(path: str) -> dict[str, Any]: with open(path, encoding='utf-8') as f: return tomllib.loads(f.read()) +@dataclass +class VariantProviderConfig: + requires: list[str] + plugin_api: str | None = None + enable_if: str | None = None + optional: bool = False + + @classmethod + def from_dict(cls, data: dict): + """Creates an instance of VariantProviderConfig from a dictionary.""" + # Convert hyphenated keys to underscored keys + data = {key.replace("-", "_"): value for key, value in data.items()} + + # Create an instance of VariantProviderConfig + return cls(**data) + + def validate(self): + """Validates the VariantProviderConfig instance.""" + if not self.requires: + raise ValueError("Requires list cannot be empty") + + +@dataclass +class VariantConfig: + vhash: str + properties: list[str] + default_priorities: dict[str, list[str]] + providers: dict[str, VariantProviderConfig] + + @classmethod + def from_dict(cls, data: dict, vprops: list[str] | None, + variant_label: str | None): + """Creates an instance of VariantConfig from a dictionary.""" + data = data.copy() + + if vprops is None: + data["vhash"] = None + data["properties"] = None + + + elif len(vprops) == 0: + data["vhash"] = "0" * VARIANT_HASH_LEN + data["properties"] = [] + + else: + # Normalizing + _vprops = [ + [el.strip() for el in vprop.split("::")] + for vprop in vprops + ] + for vprop in _vprops: + assert len(vprop) == 3, f"Invalid variant property: {vprop}" + + data["properties"] = [" :: ".join(vprop) for vprop in sorted(_vprops)] + + hash_object = hashlib.sha256() + for vprop in data["properties"]: + hash_object.update(f"{vprop}\n".encode()) + data["vhash"] = hash_object.hexdigest()[:VARIANT_HASH_LEN] + + if variant_label is not None: + data["vhash"] = variant_label + + # Convert hyphenated keys to underscored keys + data = {key.replace("-", "_"): value for key, value in data.items()} + + # Convert providers to VariantProviderConfig instances + data["providers"] = { + provider: VariantProviderConfig.from_dict(provider_data) + for provider, provider_data in data["providers"].items() + } + + # Create an instance of VariantConfig + return cls(**data) + + def validate(self): + """Validates the VariantConfig instance.""" + for namespace in self.default_priorities["namespace"]: + if namespace not in self.providers: + raise ValueError( + f"Namespace '{namespace}' is not defined in the variant providers" + ) + + for provider_cfg in self.providers.values(): + provider_cfg.validate() + + class ProjectMetadata(Generic[PluginManagerBound]): def __init__( self, @@ -58,6 +150,10 @@ def __init__( self._version: str | None = None self._project_file: str | None = None + self.variant_hash: str | None = None + self.variant_config: VariantConfig | None = None + self._variant_config_data: dict[str, Any] | None = None + # App already loaded config if config is not None and root is not None: self._project_file = os.path.join(root, 'pyproject.toml') @@ -126,6 +222,13 @@ def dynamic(self) -> list[str]: return self._dynamic + @property + def variant_config_data(self) -> dict[str, Any]: + """Variant configuration data fetched from pyproject.toml""" + if self._variant_config_data is None: + self._variant_config_data = self.config.get('variant', {}) + return self._variant_config_data + @property def name(self) -> str: # Duplicate the name parsing here for situations where it's diff --git a/ruff_defaults.toml b/ruff_defaults.toml index 9acc585a3..dacc7bb35 100644 --- a/ruff_defaults.toml +++ b/ruff_defaults.toml @@ -500,7 +500,6 @@ select = [ "S317", "S318", "S319", - "S320", "S321", "S323", "S324", @@ -629,7 +628,6 @@ select = [ "UP035", "UP036", "UP037", - "UP038", "UP039", "UP040", "UP041", diff --git a/src/hatch/cli/build/__init__.py b/src/hatch/cli/build/__init__.py index 7b5c63f69..8e57fe4da 100644 --- a/src/hatch/cli/build/__init__.py +++ b/src/hatch/cli/build/__init__.py @@ -49,8 +49,32 @@ ), ) @click.option('--clean-only', is_flag=True, hidden=True) +@click.option( + '-p', + '--variant-property', + 'variant_props', + multiple=True, + help=( + "Variant Properties to add to the Wheel Variant, can be repeated as many " + "times as needed" + ), +) +@click.option( + '--null-variant', + 'variant_null', + is_flag=True, + help='Make the variant a `null variant` - no variant property.', +) +@click.option( + '--variant-label', + 'variant_label', + help='Use a custom variant label (the default is variant hash)', +) @click.pass_obj -def build(app: Application, location, targets, hooks_only, no_hooks, ext, clean, clean_hooks_after, clean_only): +def build( + app: Application, location, targets, hooks_only, no_hooks, ext, clean, clean_hooks_after, clean_only, + variant_props, variant_null, variant_label, +): """Build a project.""" app.ensure_environment_plugin_dependencies() @@ -85,6 +109,7 @@ def build(app: Application, location, targets, hooks_only, no_hooks, ext, clean, app.display_header(target_name) if build_backend != BUILD_BACKEND: + # TODO(hcho3): Should we pass variant flags to non-Hatchling backend?? if target_name == 'sdist': directory = build_dir or app.project.location / DEFAULT_BUILD_DIRECTORY directory.ensure_dir_exists() @@ -103,6 +128,13 @@ def build(app: Application, location, targets, hooks_only, no_hooks, ext, clean, ) else: command = ['python', '-u', '-m', 'hatchling', 'build', '--target', target] + # Pass variant flags to Hatchling + for prop in variant_props: + command.extend(['-p', prop]) + if variant_null: + command.append('--null-variant') + if variant_label: + command.extend(['--variant-label', variant_label]) # We deliberately pass the location unchanged so that absolute paths may be non-local # and reflect wherever builds actually take place