From 0b3b25219c942967baafbb2459937875151badd8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 8 Dec 2025 22:15:30 +0900 Subject: [PATCH 1/3] fixup the target_platform setting and test locally --- MODULE.bazel | 5 +++++ docs/BUILD.bazel | 2 ++ python/private/pypi/hub_builder.bzl | 12 +++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b909124d11..ef7499eb0c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -305,6 +305,11 @@ dev_pip = use_extension( parallel_download = False, python_version = python_version, requirements_lock = "//docs:requirements.txt", + # Ensure that we are setting up the following platforms + target_platforms = [ + "{os}_{arch}", + "{os}_{arch}_freethreaded", + ], ) for python_version in [ "3.9", diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index d7748d35a4..55d8f75a74 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -165,6 +165,8 @@ sphinx_build_binary( config_settings = { labels.BOOTSTRAP_IMPL: "script", labels.VENVS_SITE_PACKAGES: "yes", + labels.PY_FREETHREADED: "yes", + labels.PYTHON_VERSION: "3.14", }, target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [ diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 3a1a3b07fe..84c52cf2a1 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -137,13 +137,19 @@ def _pip_parse(self, module_ctx, pip_attr): return default_cross_setup = _set_get_index_urls(self, pip_attr) + target_platforms = pip_attr.target_platforms + if default_cross_setup and not target_platforms: + # TODO @aignas 2025-12-06: use target_platforms always even when the + # get_index_urls is set. + target_platforms = [] + elif not target_platforms: + target_platforms = ["{os}_{arch}"] + self._platforms[python_version] = _platforms( module_ctx, python_version = full_python_version, config = self._config, - # FIXME @aignas 2025-12-06: should we have this behaviour? - # TODO @aignas 2025-12-06: use target_platforms always even when the get_index_urls is set. - target_platforms = [] if default_cross_setup else pip_attr.target_platforms, + target_platforms = target_platforms, ) _add_group_map(self, pip_attr.experimental_requirement_cycles) _add_extra_aliases(self, pip_attr.extra_hub_aliases) From ee29882012882ec5e2d0fd0ab92cd8470d57bc5a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 9 Dec 2025 08:08:39 +0900 Subject: [PATCH 2/3] fixup --- python/private/pypi/hub_builder.bzl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 84c52cf2a1..ff6416e1cb 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -137,19 +137,13 @@ def _pip_parse(self, module_ctx, pip_attr): return default_cross_setup = _set_get_index_urls(self, pip_attr) - target_platforms = pip_attr.target_platforms - if default_cross_setup and not target_platforms: - # TODO @aignas 2025-12-06: use target_platforms always even when the - # get_index_urls is set. - target_platforms = [] - elif not target_platforms: - target_platforms = ["{os}_{arch}"] - self._platforms[python_version] = _platforms( module_ctx, python_version = full_python_version, config = self._config, - target_platforms = target_platforms, + # TODO @aignas 2025-12-09: flip or part to default to 'os_arch' after + # VERSION_NEXT_FEATURE is released + target_platforms = pip_attr.target_platforms or ([] if default_cross_setup else ["{os}_{arch}"]), ) _add_group_map(self, pip_attr.experimental_requirement_cycles) _add_extra_aliases(self, pip_attr.extra_hub_aliases) From cd11e6991258d85f71bdc5da07639859faea13b2 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 14 Dec 2025 13:50:16 +0900 Subject: [PATCH 3/3] fix the majority of the tests --- python/private/pypi/extension.bzl | 2 +- python/private/pypi/hub_builder.bzl | 3 +- .../pypi/requirements_files_by_platform.bzl | 2 + tests/pypi/extension/extension_tests.bzl | 8 +- tests/pypi/hub_builder/hub_builder_tests.bzl | 444 ++++++++---------- 5 files changed, 206 insertions(+), 253 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2a6d43f837..3d7985ab3f 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -665,7 +665,7 @@ EXPERIMENTAL: this may be removed without notice. """, ), "target_platforms": attr.string_list( - default = ["{os}_{arch}"], + default = [], doc = """\ The list of platforms for which we would evaluate the requirements files. If you need to be able to only evaluate for a particular platform (e.g. "linux_x86_64"), then put it in here. diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index ff6416e1cb..97e0a111b2 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -142,7 +142,8 @@ def _pip_parse(self, module_ctx, pip_attr): python_version = full_python_version, config = self._config, # TODO @aignas 2025-12-09: flip or part to default to 'os_arch' after - # VERSION_NEXT_FEATURE is released + # VERSION_NEXT_FEATURE is released and set the default of the `target_platforms` attribute + # to `{os}_{arch}`. target_platforms = pip_attr.target_platforms or ([] if default_cross_setup else ["{os}_{arch}"]), ) _add_group_map(self, pip_attr.experimental_requirement_cycles) diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl index 2027b41594..c0fc93a0fe 100644 --- a/python/private/pypi/requirements_files_by_platform.bzl +++ b/python/private/pypi/requirements_files_by_platform.bzl @@ -144,6 +144,8 @@ def requirements_files_by_platform( input_platforms = platforms default_platforms = [_platform(p, python_version) for p in platforms] + if logger: + logger.debug(lambda: "Input platforms: {}".format(input_platforms)) if platforms_from_args: lock_files = [ diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b1e363bc7b..924796c703 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -22,12 +22,12 @@ load(":pip_parse.bzl", _parse = "pip_parse") _tests = [] -def _mock_mctx(*modules, environ = {}, read = None): +def _mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", environ = {}, read = None): return struct( os = struct( environ = environ, - name = "unittest", - arch = "exotic", + name = os_name, + arch = arch_name, ), read = read or (lambda _: """\ simple==0.0.1 \ @@ -148,6 +148,8 @@ def _test_simple(env): ), ], ), + os_name = "linux", + arch_name = "x86_64", ), available_interpreters = { "python_3_15_host": "unit_test_interpreter_target", diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index e267f4ca34..bf21dcacfa 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -25,12 +25,12 @@ load("//tests/pypi/extension:pip_parse.bzl", _parse = "pip_parse") _tests = [] -def _mock_mctx(os = "unittest", arch = "exotic", environ = {}, read = None): +def _mock_mctx(os_name = "unittest", arch_name = "exotic", environ = {}, read = None): return struct( os = struct( environ = environ, - name = os, - arch = arch, + name = os_name, + arch = arch_name, ), read = read or (lambda _: """\ simple==0.0.1 \ @@ -114,7 +114,10 @@ def hub_builder( def _test_simple(env): builder = hub_builder(env) builder.pip_parse( - _mock_mctx(), + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + ), _parse( hub_name = "pypi", python_version = "3.15", @@ -147,118 +150,94 @@ def _test_simple(env): _tests.append(_test_simple) def _test_simple_multiple_requirements(env): - builder = hub_builder(env) - builder.pip_parse( - _mock_mctx( - read = lambda x: { - "darwin.txt": "simple==0.0.2 --hash=sha256:deadb00f", - "win.txt": "simple==0.0.1 --hash=sha256:deadbeef", - }[x], - ), - _parse( - hub_name = "pypi", - python_version = "3.15", - requirements_darwin = "darwin.txt", - requirements_windows = "win.txt", - ), - ) - pypi = builder.build() + sub_tests = { + ("osx", "aarch64"): "simple==0.0.2 --hash=sha256:deadb00f", + ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + } + for (host_os, host_arch), want_requirement in sub_tests.items(): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + read = lambda x: { + "darwin.txt": "simple==0.0.2 --hash=sha256:deadb00f", + "win.txt": "simple==0.0.1 --hash=sha256:deadbeef", + }[x], + os_name = host_os, + arch_name = host_arch, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_darwin = "darwin.txt", + requirements_windows = "win.txt", + ), + ) + pypi = builder.build() - pypi.exposed_packages().contains_exactly(["simple"]) - pypi.group_map().contains_exactly({}) - pypi.whl_map().contains_exactly({ - "simple": { - "pypi_315_simple_osx_aarch64": [ - whl_config_setting( - target_platforms = [ - "cp315_osx_aarch64", - ], - version = "3.15", - ), - ], - "pypi_315_simple_windows_aarch64": [ - whl_config_setting( - target_platforms = [ - "cp315_windows_aarch64", - ], - version = "3.15", - ), - ], - }, - }) - pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.2 --hash=sha256:deadb00f", - }, - "pypi_315_simple_windows_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1 --hash=sha256:deadbeef", - }, - }) - pypi.extra_aliases().contains_exactly({}) + pypi.exposed_packages().contains_exactly(["simple"]) + pypi.group_map().contains_exactly({}) + pypi.whl_map().contains_exactly({ + "simple": { + "pypi_315_simple": [ + whl_config_setting(version = "3.15"), + ], + }, + }) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": want_requirement, + }, + }) + pypi.extra_aliases().contains_exactly({}) _tests.append(_test_simple_multiple_requirements) def _test_simple_extras_vs_no_extras(env): - builder = hub_builder(env) - builder.pip_parse( - _mock_mctx( - read = lambda x: { - "darwin.txt": "simple[foo]==0.0.1 --hash=sha256:deadbeef", - "win.txt": "simple==0.0.1 --hash=sha256:deadbeef", - }[x], - ), - _parse( - hub_name = "pypi", - python_version = "3.15", - requirements_darwin = "darwin.txt", - requirements_windows = "win.txt", - ), - ) - pypi = builder.build() + sub_tests = { + ("osx", "aarch64"): "simple[foo]==0.0.1 --hash=sha256:deadbeef", + ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + } + for (host_os, host_arch), want_requirement in sub_tests.items(): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + read = lambda x: { + "darwin.txt": "simple[foo]==0.0.1 --hash=sha256:deadbeef", + "win.txt": "simple==0.0.1 --hash=sha256:deadbeef", + }[x], + os_name = host_os, + arch_name = host_arch, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_darwin = "darwin.txt", + requirements_windows = "win.txt", + ), + ) + pypi = builder.build() - pypi.exposed_packages().contains_exactly(["simple"]) - pypi.group_map().contains_exactly({}) - pypi.whl_map().contains_exactly({ - "simple": { - "pypi_315_simple_osx_aarch64": [ - whl_config_setting( - target_platforms = [ - "cp315_osx_aarch64", - ], - version = "3.15", - ), - ], - "pypi_315_simple_windows_aarch64": [ - whl_config_setting( - target_platforms = [ - "cp315_windows_aarch64", - ], - version = "3.15", - ), - ], - }, - }) - pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple[foo]==0.0.1 --hash=sha256:deadbeef", - }, - "pypi_315_simple_windows_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1 --hash=sha256:deadbeef", - }, - }) - pypi.extra_aliases().contains_exactly({}) + pypi.exposed_packages().contains_exactly(["simple"]) + pypi.group_map().contains_exactly({}) + pypi.whl_map().contains_exactly({ + "simple": { + "pypi_315_simple": [ + whl_config_setting(version = "3.15"), + ], + }, + }) + pypi.whl_libraries().contains_exactly({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": want_requirement, + }, + }) + pypi.extra_aliases().contains_exactly({}) _tests.append(_test_simple_extras_vs_no_extras) @@ -358,6 +337,8 @@ simple==0.0.1 --hash=sha256:deadbeef old-package==0.0.1 --hash=sha256:deadbaaf """, }[x], + os_name = "linux", + arch_name = "amd64", ), _parse( hub_name = "pypi", @@ -373,6 +354,8 @@ simple==0.0.2 --hash=sha256:deadb00f new-package==0.0.1 --hash=sha256:deadb00f2 """, }[x], + os_name = "linux", + arch_name = "amd64", ), _parse( hub_name = "pypi", @@ -387,28 +370,20 @@ new-package==0.0.1 --hash=sha256:deadb00f2 pypi.whl_map().contains_exactly({ "new_package": { "pypi_316_new_package": [ - whl_config_setting( - version = "3.16", - ), + whl_config_setting(version = "3.16"), ], }, "old_package": { "pypi_315_old_package": [ - whl_config_setting( - version = "3.15", - ), + whl_config_setting(version = "3.15"), ], }, "simple": { "pypi_315_simple": [ - whl_config_setting( - version = "3.15", - ), + whl_config_setting(version = "3.15"), ], "pypi_316_simple": [ - whl_config_setting( - version = "3.16", - ), + whl_config_setting(version = "3.16"), ], }, }) @@ -443,75 +418,62 @@ new-package==0.0.1 --hash=sha256:deadb00f2 _tests.append(_test_simple_multiple_python_versions) def _test_simple_with_markers(env): - builder = hub_builder( - env, - evaluate_markers_fn = lambda _, requirements, **__: { - key: [ - platform - for platform in platforms - if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key) - ] - for key, platforms in requirements.items() - }, - ) - builder.pip_parse( - _mock_mctx( - read = lambda x: { - "universal.txt": """\ -torch==2.4.1+cpu ; platform_machine == 'x86_64' -torch==2.4.1 ; platform_machine != 'x86_64' \ - --hash=sha256:deadbeef -""", - }[x], - ), - _parse( - hub_name = "pypi", - python_version = "3.15", - requirements_lock = "universal.txt", - ), - ) - pypi = builder.build() + sub_tests = { + ("osx", "aarch64"): "torch==2.4.1 --hash=sha256:deadbeef", + ("linux", "x86_64"): "torch==2.4.1+cpu", + } + for (host_os, host_arch), want_requirement in sub_tests.items(): + builder = hub_builder( + env, + evaluate_markers_fn = lambda _, requirements, **__: { + key: [ + platform + for platform in platforms + if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key) + ] + for key, platforms in requirements.items() + }, + ) + builder.pip_parse( + _mock_mctx( + read = lambda x: { + "universal.txt": """\ + torch==2.4.1+cpu ; platform_machine == 'x86_64' + torch==2.4.1 ; platform_machine != 'x86_64' \ + --hash=sha256:deadbeef + """, + }[x], + os_name = host_os, + arch_name = host_arch, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "universal.txt", + ), + ) + pypi = builder.build() - pypi.exposed_packages().contains_exactly(["torch"]) - pypi.group_map().contains_exactly({}) - pypi.whl_map().contains_exactly({ - "torch": { - "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": [ - whl_config_setting( - target_platforms = [ - "cp315_linux_aarch64", - "cp315_osx_aarch64", - "cp315_windows_aarch64", - ], - version = "3.15", - ), - ], - "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": [ - whl_config_setting( - target_platforms = [ - "cp315_linux_x86_64", - "cp315_linux_x86_64_freethreaded", - ], - version = "3.15", - ), - ], - }, - }) - pypi.whl_libraries().contains_exactly({ - "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "torch==2.4.1 --hash=sha256:deadbeef", - }, - "pypi_315_torch_linux_x86_64_linux_x86_64_freethreaded": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "torch==2.4.1+cpu", - }, - }) - pypi.extra_aliases().contains_exactly({}) + pypi.exposed_packages().contains_exactly(["torch"]) + pypi.group_map().contains_exactly({}) + pypi.whl_map().contains_exactly({ + "torch": { + "pypi_315_torch": [ + whl_config_setting( + version = "3.15", + ), + ], + }, + }) + pypi.whl_libraries().contains_exactly({ + "pypi_315_torch": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": want_requirement, + }, + }) + pypi.extra_aliases().contains_exactly({}) _tests.append(_test_simple_with_markers) @@ -1079,66 +1041,51 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): - builder = hub_builder( - env, - ) - builder.pip_parse( - _mock_mctx( - read = lambda x: { - "universal.txt": """\ + sub_tests = { + ("osx", "aarch64"): "optimum[onnxruntime]==1.17.1", + ("linux", "aarch64"): "optimum[onnxruntime-gpu]==1.17.1", + } + for (host_os, host_arch), want_requirement in sub_tests.items(): + builder = hub_builder( + env, + ) + builder.pip_parse( + _mock_mctx( + read = lambda x: { + "universal.txt": """\ optimum[onnxruntime]==1.17.1 ; sys_platform == 'darwin' optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' """, - }[x], - ), - _parse( - hub_name = "pypi", - python_version = "3.15", - requirements_lock = "universal.txt", - ), - ) - pypi = builder.build() + }[x], + os_name = host_os, + arch_name = host_arch, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "universal.txt", + ), + ) + pypi = builder.build() - # FIXME @aignas 2025-09-07: we should expose the `optimum` package - pypi.exposed_packages().contains_exactly([]) - pypi.group_map().contains_exactly({}) - pypi.whl_map().contains_exactly({ - "optimum": { - "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": [ - whl_config_setting( - version = "3.15", - target_platforms = [ - "cp315_linux_aarch64", - "cp315_linux_x86_64", - "cp315_linux_x86_64_freethreaded", - ], - ), - ], - "pypi_315_optimum_osx_aarch64": [ - whl_config_setting( - version = "3.15", - target_platforms = [ - "cp315_osx_aarch64", - ], - ), - ], - }, - }) - pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_linux_aarch64_linux_x86_64_linux_x86_64_freethreaded": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", - }, - "pypi_315_optimum_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime]==1.17.1", - }, - }) - pypi.extra_aliases().contains_exactly({}) + pypi.exposed_packages().contains_exactly(["optimum"]) + pypi.group_map().contains_exactly({}) + pypi.whl_map().contains_exactly({ + "optimum": { + "pypi_315_optimum": [ + whl_config_setting(version = "3.15"), + ], + }, + }) + pypi.whl_libraries().contains_exactly({ + "pypi_315_optimum": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": want_requirement, + }, + }) + pypi.extra_aliases().contains_exactly({}) _tests.append(_test_optimum_sys_platform_extra) @@ -1181,6 +1128,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' hub_name = "pypi", python_version = "3.15", requirements_lock = "universal.txt", + target_platforms = ["mylinuxx86_64", "myosxaarch64"], ), ) pypi = builder.build() @@ -1253,8 +1201,8 @@ def _test_pipstar_platforms_limit(env): ) builder.pip_parse( _mock_mctx( - os = "linux", - arch = "amd64", + os_name = "linux", + arch_name = "amd64", read = lambda x: { "universal.txt": """\ optimum[onnxruntime]==1.17.1 ; sys_platform == 'darwin'