From 7aab0432fa9318211baffac0f9b9e58514478368 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 13 Mar 2025 12:47:33 +0000 Subject: [PATCH 01/11] Basic support for uv --- .github/workflows/main.yml | 22 +++++++++++++++------ integration_tests/Makefile | 2 +- integration_tests/src/numpy.sh | 2 +- pyodide_build/pypabuild.py | 5 +++-- pyodide_build/recipe/graph_builder.py | 4 +++- pyodide_build/uv_helper.py | 28 +++++++++++++++++++++++++++ pyodide_build/xbuildenv.py | 22 +++++++++++++++------ pyproject.toml | 3 +++ 8 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 pyodide_build/uv_helper.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index acdf38ae..98a29bb5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,7 +86,13 @@ jobs: strategy: fail-fast: false matrix: - task: [test-recipe, test-src, test-integration-marker] + task: [ + {name: test-recipe, installer: pip}, + {name: test-src, installer: pip}, + {name: test-recipe, installer: uv}, + {name: test-src, installer: uv}, + {name: test-integration-marker, installer: pip}, # installer doesn't matter + ] if: needs.check-integration-test-trigger.outputs.run-integration-test steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -107,7 +113,7 @@ jobs: - name: Install the package run: | python -m pip install --upgrade pip - python -m pip install -e ."[test]" + python -m pip install -e ."[test,uv]" - name: Install xbuildenv run: | @@ -124,19 +130,23 @@ jobs: run: echo "CORES=$(nproc)" >> $GITHUB_OUTPUT - name: Run tests marked with integration - if: matrix.task == 'test-integration-marker' + if: matrix.task.name == 'test-integration-marker' run: pytest --junitxml=test-results/junit.xml --cov=pyodide-build pyodide_build -m "integration" - name: Run the recipe integration tests (${{ matrix.task }}) - if: matrix.task != 'test-integration-marker' + if: matrix.task.name != 'test-integration-marker' env: PYODIDE_JOBS: ${{ steps.get-cores.outputs.CORES }} working-directory: integration_tests - run: make ${{ matrix.task }} + run: | + if [[ "${{ matrix.task.installer }}" == "uv" ]]; then + export UV_RUN_PREFIX="uv run" + fi + make ${{ matrix.task }} - name: Upload coverage for tests marked with integration uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - if: matrix.task == 'test-integration-marker' + if: matrix.task.name == 'test-integration-marker' with: name: coverage-from-integration path: .coverage diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 0bb7c8bf..c597683b 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -6,7 +6,7 @@ all: test-recipe: check @echo "... Running integration tests for building recipes" - pyodide build-recipes --recipe-dir=recipes --force-rebuild "*" + $(UV_RUN_PREFIX) pyodide build-recipes --recipe-dir=recipes --force-rebuild "*" @echo "... Passed" diff --git a/integration_tests/src/numpy.sh b/integration_tests/src/numpy.sh index 3b01adaa..1abe6b44 100755 --- a/integration_tests/src/numpy.sh +++ b/integration_tests/src/numpy.sh @@ -10,4 +10,4 @@ tar -xf numpy-${VERSION}.tar.gz cd numpy-${VERSION} MESON_CROSS_FILE=$(pyodide config get meson_cross_file) -pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" +${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index e9c4e437..0f6ced0a 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -15,7 +15,7 @@ from build.env import DefaultIsolatedEnv from packaging.requirements import Requirement -from pyodide_build import _f2c_fixes, common, pywasmcross +from pyodide_build import _f2c_fixes, common, pywasmcross, uv_helper from pyodide_build.build_env import ( get_build_flag, get_hostsitepackages, @@ -143,7 +143,8 @@ def _build_in_isolated_env( # It will be left in the /tmp folder and can be inspected or entered as # needed. # _DefaultIsolatedEnv.__exit__ = lambda *args: None - with _DefaultIsolatedEnv() as env: + installer = "uv" if uv_helper.should_use_uv() else "pip" + with _DefaultIsolatedEnv(installer=installer) as env: env = cast(_DefaultIsolatedEnv, env) builder = _ProjectBuilder.from_isolated_env( env, diff --git a/pyodide_build/recipe/graph_builder.py b/pyodide_build/recipe/graph_builder.py index b408dd08..c6117e7f 100755 --- a/pyodide_build/recipe/graph_builder.py +++ b/pyodide_build/recipe/graph_builder.py @@ -27,7 +27,7 @@ from rich.spinner import Spinner from rich.table import Table -from pyodide_build import build_env +from pyodide_build import build_env, uv_helper from pyodide_build.build_env import BuildArgs from pyodide_build.common import ( download_and_unpack_archive, @@ -122,8 +122,10 @@ def needs_rebuild(self, build_dir: Path) -> bool: return needs_rebuild(self.pkgdir, self.build_path(build_dir), self.meta.source) def build(self, build_args: BuildArgs, build_dir: Path) -> None: + run_prefix = [uv_helper.find_uv_bin(), "run"] if uv_helper.should_use_uv() else [] p = subprocess.run( [ + *run_prefix, "pyodide", "build-recipes-no-deps", self.name, diff --git a/pyodide_build/uv_helper.py b/pyodide_build/uv_helper.py new file mode 100644 index 00000000..284463bc --- /dev/null +++ b/pyodide_build/uv_helper.py @@ -0,0 +1,28 @@ +import functools +import shutil +import os + + +@functools.cache +def find_uv_bin() -> str | None: + """ + Check if the uv executable is available. + + If the uv executable is available, return the path to the executable. + Otherwise, return None. + """ + try: + import uv + + return uv.find_uv_bin() + except (ModuleNotFoundError, FileNotFoundError): + return shutil.which('uv') + + return None + + +def should_use_uv() -> bool: + # UV environ is set to the uv executable path when the script is called with the uv executable. + uv_environ = os.environ.get("UV") + # double check by comparing the uv executable path with the one found by the uv package. + return find_uv_bin() == uv_environ diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index dbada059..5462a435 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -2,10 +2,11 @@ import shutil import subprocess from pathlib import Path +import sys from pyodide_lock import PyodideLockSpec -from pyodide_build import build_env +from pyodide_build import build_env, uv_helper from pyodide_build.common import download_and_unpack_archive from pyodide_build.create_package_index import create_package_index from pyodide_build.logger import logger @@ -265,15 +266,24 @@ def _install_cross_build_packages( """ host_site_packages = self._host_site_packages_dir(xbuildenv_pyodide_root) host_site_packages.mkdir(exist_ok=True, parents=True) + + install_prefix = [ + uv_helper.find_uv_bin(), + "pip", + "install", + ] if uv_helper.should_use_uv() else [ + "pip", + "install", + "--no-user", + ] + result = subprocess.run( [ - "pip", - "install", - "--no-user", - "-t", - str(host_site_packages), + *install_prefix, "-r", str(xbuildenv_root / "requirements.txt"), + "--target", + str(host_site_packages), ], capture_output=True, encoding="utf8", diff --git a/pyproject.toml b/pyproject.toml index d327c680..0b183d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,9 @@ test = [ "pytest-cov", "types-requests", ] +uv = [ + "build[uv]~=1.2.0", +] [tool.hatch.version] source = "vcs" From ec25ecd079de91a7cb49eedbeb9bb72f27806519 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 13 Mar 2025 12:49:02 +0000 Subject: [PATCH 02/11] lint [integration] --- pyodide_build/recipe/graph_builder.py | 4 +++- pyodide_build/uv_helper.py | 4 ++-- pyodide_build/xbuildenv.py | 23 +++++++++++++---------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/pyodide_build/recipe/graph_builder.py b/pyodide_build/recipe/graph_builder.py index c6117e7f..8f107148 100755 --- a/pyodide_build/recipe/graph_builder.py +++ b/pyodide_build/recipe/graph_builder.py @@ -122,7 +122,9 @@ def needs_rebuild(self, build_dir: Path) -> bool: return needs_rebuild(self.pkgdir, self.build_path(build_dir), self.meta.source) def build(self, build_args: BuildArgs, build_dir: Path) -> None: - run_prefix = [uv_helper.find_uv_bin(), "run"] if uv_helper.should_use_uv() else [] + run_prefix = ( + [uv_helper.find_uv_bin(), "run"] if uv_helper.should_use_uv() else [] + ) p = subprocess.run( [ *run_prefix, diff --git a/pyodide_build/uv_helper.py b/pyodide_build/uv_helper.py index 284463bc..7f04a88b 100644 --- a/pyodide_build/uv_helper.py +++ b/pyodide_build/uv_helper.py @@ -1,6 +1,6 @@ import functools -import shutil import os +import shutil @functools.cache @@ -16,7 +16,7 @@ def find_uv_bin() -> str | None: return uv.find_uv_bin() except (ModuleNotFoundError, FileNotFoundError): - return shutil.which('uv') + return shutil.which("uv") return None diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index 5462a435..f970816f 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -2,7 +2,6 @@ import shutil import subprocess from pathlib import Path -import sys from pyodide_lock import PyodideLockSpec @@ -267,15 +266,19 @@ def _install_cross_build_packages( host_site_packages = self._host_site_packages_dir(xbuildenv_pyodide_root) host_site_packages.mkdir(exist_ok=True, parents=True) - install_prefix = [ - uv_helper.find_uv_bin(), - "pip", - "install", - ] if uv_helper.should_use_uv() else [ - "pip", - "install", - "--no-user", - ] + install_prefix = ( + [ + uv_helper.find_uv_bin(), + "pip", + "install", + ] + if uv_helper.should_use_uv() + else [ + "pip", + "install", + "--no-user", + ] + ) result = subprocess.run( [ From 2ffc601c55e93a0466fe9b95c7b00bff2de0e568 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 13 Mar 2025 12:51:56 +0000 Subject: [PATCH 03/11] cache emscripten --- .github/workflows/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98a29bb5..dce6e6d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,9 @@ concurrency: env: FORCE_COLOR: 3 + # Increase this value to reset cache if emscripten_version has not changed + EMSDK_CACHE_FOLDER: 'emsdk-cache' + EMSDK_CACHE_NUMBER: 0 jobs: test: @@ -120,10 +123,17 @@ jobs: pyodide xbuildenv install echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV + - name: Cache emsdk + uses: actions/cache@v4 + with: + path: ${{ env.EMSDK_CACHE_FOLDER }} + key: ${{ env.EMSDK_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }} + - name: Install Emscripten uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: ${{env.EMSDK_CACHE_FOLDER}} - name: Get number of cores on the runner id: get-cores From 3b7981b0b99123a69b58163fa222ac21c534a969 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 13 Mar 2025 12:52:23 +0000 Subject: [PATCH 04/11] ruff [integration] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dce6e6d8..ea9ab655 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -128,7 +128,7 @@ jobs: with: path: ${{ env.EMSDK_CACHE_FOLDER }} key: ${{ env.EMSDK_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }} - + - name: Install Emscripten uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14 with: From 4d1c86460d5cdab0da31bc8c780be0e9ae15361e Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Thu, 13 Mar 2025 12:52:49 +0000 Subject: [PATCH 05/11] typo --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea9ab655..b0a877aa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,7 +152,7 @@ jobs: if [[ "${{ matrix.task.installer }}" == "uv" ]]; then export UV_RUN_PREFIX="uv run" fi - make ${{ matrix.task }} + make ${{ matrix.task.name }} - name: Upload coverage for tests marked with integration uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 From 87c0c3b13c5d42f837bb82b928cc655c09898e59 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Sat, 15 Mar 2025 08:04:29 +0000 Subject: [PATCH 06/11] check none --- pyodide_build/uv_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyodide_build/uv_helper.py b/pyodide_build/uv_helper.py index 7f04a88b..52958e0f 100644 --- a/pyodide_build/uv_helper.py +++ b/pyodide_build/uv_helper.py @@ -25,4 +25,4 @@ def should_use_uv() -> bool: # UV environ is set to the uv executable path when the script is called with the uv executable. uv_environ = os.environ.get("UV") # double check by comparing the uv executable path with the one found by the uv package. - return find_uv_bin() == uv_environ + return uv_environ and uv_environ == find_uv_bin() From 02ed7d79026fb4851d47688e66c8681eb9d32de8 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Sat, 15 Mar 2025 08:06:37 +0000 Subject: [PATCH 07/11] changelog [integration] --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1c3468..9ab63855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added basic support for uv. `uv run pyodide` will now work. + [#132](https://github.com/pyodide/pyodide-build/pull/132) + ### Changed - The Rust toolchain version has been updated to `nightly-2025-01-18`. From eff8a227455a362658ef13bba61c778a608edfa2 Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Tue, 25 Mar 2025 19:54:44 +0900 Subject: [PATCH 08/11] Apply suggestions from code review Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- CHANGELOG.md | 2 +- pyodide_build/xbuildenv.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab63855..e0f05a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added basic support for uv. `uv run pyodide` will now work. +- Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` in `uv`-managed virtual environments will now work. [#132](https://github.com/pyodide/pyodide-build/pull/132) ### Changed diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index dbe8e9eb..28e06182 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -293,6 +293,8 @@ def _install_cross_build_packages( ] if uv_helper.should_use_uv() else [ + sys.executable, + "-m" "pip", "install", "--no-user", From c31256a17b04faa7ee3d72f2db008cc1bfc8238d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:55:42 +0000 Subject: [PATCH 09/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyodide_build/xbuildenv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index 28e06182..2660de78 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -294,8 +294,7 @@ def _install_cross_build_packages( if uv_helper.should_use_uv() else [ sys.executable, - "-m" - "pip", + "-m" "pip", "install", "--no-user", ] From 45850d953e6cdc8e46c72873a17c8a15880e343d Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Tue, 25 Mar 2025 11:17:09 +0000 Subject: [PATCH 10/11] Fix test --- pyodide_build/tests/test_xbuildenv.py | 18 +++++++++++++----- pyodide_build/xbuildenv.py | 4 +++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pyodide_build/tests/test_xbuildenv.py b/pyodide_build/tests/test_xbuildenv.py index fca03260..89396629 100644 --- a/pyodide_build/tests/test_xbuildenv.py +++ b/pyodide_build/tests/test_xbuildenv.py @@ -16,7 +16,7 @@ def monkeypatch_subprocess_run_pip(monkeypatch): orig_run = subprocess.run def monkeypatch_func(cmds, *args, **kwargs): - if cmds[0] == "pip": + if cmds[0] == "pip" or cmds[0:3] == [sys.executable, "-m", "pip"]: called_with.extend(cmds) return subprocess.CompletedProcess(cmds, 0, "", "") else: @@ -289,12 +289,20 @@ def test_install_cross_build_packages( xbuildenv_pyodide_root = xbuildenv_root / "pyodide-root" manager._install_cross_build_packages(xbuildenv_root, xbuildenv_pyodide_root) - assert len(pip_called_with) == 7 - assert pip_called_with[0:4] == ["pip", "install", "--no-user", "-t"] - assert pip_called_with[4].startswith( + assert len(pip_called_with) == 9 + assert pip_called_with[0:8] == [ + sys.executable, + "-m", + "pip", + "install", + "--no-user", + "-r", + str(xbuildenv_root / "requirements.txt"), + "--target", + ] + assert pip_called_with[8].startswith( str(xbuildenv_pyodide_root) ) # hostsitepackages - assert pip_called_with[5:7] == ["-r", str(xbuildenv_root / "requirements.txt")] hostsitepackages = manager._host_site_packages_dir(xbuildenv_pyodide_root) assert hostsitepackages.exists() diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index 2660de78..3d1dc8cc 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -1,6 +1,7 @@ import json import shutil import subprocess +import sys from pathlib import Path from pyodide_lock import PyodideLockSpec @@ -294,7 +295,8 @@ def _install_cross_build_packages( if uv_helper.should_use_uv() else [ sys.executable, - "-m" "pip", + "-m", + "pip", "install", "--no-user", ] From e46ef6b1003720c7063d724c7e4b0ec455d4caca Mon Sep 17 00:00:00 2001 From: Gyeongjae Choi Date: Tue, 25 Mar 2025 11:18:44 +0000 Subject: [PATCH 11/11] [integration]