Add support for building without isolation

.github/workflows/main.yml

@@ -105,8 +105,11 @@ jobs:
           ]
         os: [ubuntu-latest, macos-latest]
         pyodide-version: [stable]
-        # Run Pyodide minimum version testing only for the pip installer and on Linux
         include:
+        # Run no-isolation tests and Pyodide minimum version testing only
+        # for the pip installer and on Linux
+          - task: {name: test-src-no-isolation, installer: pip} # installer doesn't matter
+            os: ubuntu-latest
           - task: { name: test-recipe, installer: pip }
             os: ubuntu-latest
             pyodide-version: minimum

CHANGELOG.md

@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` in `uv`-managed virtual environments will now work.
   [#132](https://github.com/pyodide/pyodide-build/pull/132)
 
+- Added support for building without a wheel without build isolation: `pyodide build` no accepts
+  the `--no-isolation`/`-n` and/or `--skip-dependency-check`/`-x` flags to customise the wheel
+  building behaviour, similar to `pypa/build`.
+
 ### Changed
 
 - The Rust toolchain version has been updated to `nightly-2025-01-18`.

integration_tests/Makefile

@@ -23,6 +23,25 @@ test-src: check
 
 	@echo "... Passed"
 
+.PHONY: test-src-no-isolation
+test-src-no-isolation: check
+	@echo "... Running integration tests for building src with --no-isolation --skip-dependency-check"
+
+	# Some virtualenv workarounds from https://stackoverflow.com/a/24736236
+	# to make sure that we are using the right environment
+	@( \
+        set -e; \
+		python -m venv ./test_venv; \
+		. ./test_venv/bin/activate; \
+		pip install meson-python meson cython; \
+		pip install -e ../; \
+		./src/numpy_no_isolation.sh; \
+		deactivate; \
+	)
+
+	@rm -rf ./test_venv
+
+	@echo "... Passed"
 .PHONY: check
 check:
 	@echo "... Checking dependencies"
@@ -36,5 +55,9 @@ check:
 clean:
 	rm -rf .pyodide-xbuildenv*
 	rm -rf recipes/*/build
+	rm -rf test_venv
 	rm -rf src/numpy-*
+	rm -rf src/numpy-*.tar.gz
+	rm -rf numpy-*
+	rm -rf numpy-*.tar.gz
 	rm -rf dist

integration_tests/src/numpy_no_isolation.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# The same as "numpy.sh", but without the isolation, and
+# builds both NumPy and numpy-tests from a persistent
+# build directory.
+
+set -e
+
+VERSION="2.0.2"
+URL="https://files.pythonhosted.org/packages/source/n/numpy/numpy-${VERSION}.tar.gz"
+
+wget $URL
+tar -xf numpy-${VERSION}.tar.gz
+cd numpy-${VERSION}
+
+MESON_CROSS_FILE=$(pyodide config get meson_cross_file)
+
+# Build in a persistent build directory
+${UV_RUN_PREFIX} pyodide build \
+    -Csetup-args=-Dallow-noblas=true \
+    -Csetup-args=--cross-file="${MESON_CROSS_FILE}" \
+    -Cinstall-args=--tags=runtime,python-runtime,devel \
+    -Cbuild-dir="build" \
+    --no-isolation --skip-dependency-check
+
+sed -i 's/numpy/numpy-tests/g' pyproject.toml
+
+${UV_RUN_PREFIX} pyodide build \
+    -Csetup-args=-Dallow-noblas=true \
+    -Csetup-args=--cross-file="${MESON_CROSS_FILE}" \
+    -Cinstall-args=--tags=tests \
+    -Cbuild-dir="build" \
+    --no-isolation --skip-dependency-check
+
+echo "Successfully built wheels for numpy-${VERSION} and numpy-tests-${VERSION}."

pyodide_build/cli/build.py

@@ -46,6 +46,8 @@ def pypi(
     output_directory: Path,
     exports: str,
     config_settings: ConfigSettingsType,
+    isolation: bool = True,
+    skip_dependency_check: bool = False,
 ) -> Path:
     """Fetch a wheel from pypi, or build from source if none available."""
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -65,6 +67,8 @@ def pypi(
             output_directory,
             convert_exports(exports),
             config_settings,
+            isolation=isolation,
+            skip_dependency_check=skip_dependency_check,
         )
         return built_wheel
 
@@ -86,6 +90,8 @@ def url(
     output_directory: Path,
     exports: str,
     config_settings: ConfigSettingsType,
+    isolation: bool = True,
+    skip_dependency_check: bool = False,
 ) -> Path:
     """Fetch a wheel or build sdist from url."""
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -102,7 +108,12 @@ def url(
             # unzipped into subfolder
             builddir = files[0]
         wheel_path = build.run(
-            builddir, output_directory, convert_exports(exports), config_settings
+            builddir,
+            output_directory,
+            convert_exports(exports),
+            config_settings,
+            isolation=isolation,
+            skip_dependency_check=skip_dependency_check,
         )
         return wheel_path
 
@@ -112,10 +123,17 @@ def source(
     output_directory: Path,
     exports: str,
     config_settings: ConfigSettingsType,
+    isolation: bool = True,
+    skip_dependency_check: bool = False,
 ) -> Path:
     """Use pypa/build to build a Python package from source"""
     built_wheel = build.run(
-        source_location, output_directory, convert_exports(exports), config_settings
+        source_location,
+        output_directory,
+        convert_exports(exports),
+        config_settings,
+        isolation=isolation,
+        skip_dependency_check=skip_dependency_check,
     )
     return built_wheel
 
@@ -164,6 +182,20 @@ def main(
     compression_level: int = typer.Option(
         6, help="Compression level to use for the created zip file"
     ),
+    no_isolation: bool = typer.Option(
+        False,
+        "--no-isolation",
+        "-n",
+        help="Disable building the project in an isolated virtual environment. "
+        "Build dependencies must be installed separately when this option is used",
+    ),
+    skip_dependency_check: bool = typer.Option(
+        False,
+        "--skip-dependency-check",
+        "-x",
+        help="Do not check that the build dependencies are installed. This option "
+        "is only useful when used with --no-isolation.",
+    ),
     config_setting: list[str] | None = typer.Option(
         None,
         "--config-setting",
@@ -232,6 +264,8 @@ def main(
                 # dependencies? Not sure this makes sense...
                 convert_exports(exports),
                 config_settings,
+                isolation=not no_isolation,
+                skip_dependency_check=skip_dependency_check,
                 output_lockfile=output_lockfile,
             )
         except BaseException as e:
@@ -247,17 +281,43 @@ def main(
             source_location = source_location[0 : source_location.find("[")]
     if not source_location:
         # build the current folder
-        wheel = source(Path.cwd(), outpath, exports, config_settings)
+        wheel = source(
+            Path.cwd(),
+            outpath,
+            exports,
+            config_settings,
+            isolation=not no_isolation,
+            skip_dependency_check=skip_dependency_check,
+        )
     elif source_location.find("://") != -1:
-        wheel = url(source_location, outpath, exports, config_settings)
+        wheel = url(
+            source_location,
+            outpath,
+            exports,
+            config_settings,
+            isolation=not no_isolation,
+            skip_dependency_check=skip_dependency_check,
+        )
     elif Path(source_location).is_dir():
         # a folder, build it
         wheel = source(
-            Path(source_location).resolve(), outpath, exports, config_settings
+            Path(source_location).resolve(),
+            outpath,
+            exports,
+            config_settings,
+            isolation=not no_isolation,
+            skip_dependency_check=skip_dependency_check,
         )
     elif source_location.find("/") == -1:
         # try fetch or build from pypi
-        wheel = pypi(source_location, outpath, exports, config_settings)
+        wheel = pypi(
+            source_location,
+            outpath,
+            exports,
+            config_settings,
+            isolation=not no_isolation,
+            skip_dependency_check=skip_dependency_check,
+        )
     else:
         raise RuntimeError(f"Couldn't determine source type for {source_location}")
     # now build deps for wheel
@@ -271,6 +331,8 @@ def main(
                 # dependencies? Not sure this makes sense...
                 convert_exports(exports),
                 config_settings,
+                isolation=not no_isolation,
+                skip_dependency_check=skip_dependency_check,
                 output_lockfile=output_lockfile,
                 compression_level=compression_level,
             )

pyodide_build/common.py

@@ -383,6 +383,19 @@ def _get_sha256_checksum(archive: Path) -> str:
     return h.hexdigest()
 
 
+def _format_dep_chain(dep_chain: Sequence[str]) -> str:
+    return " -> ".join(dep.partition(";")[0].strip() for dep in dep_chain)
+
+
+def _format_missing_dependencies(missing: set[tuple[str, ...]]) -> str:
+    return "".join(
+        "\n\t" + dep
+        for deps in missing
+        for dep in (deps[0], _format_dep_chain(deps[1:]))
+        if dep
+    )
+
+
 def unpack_wheel(
     wheel_path: Path, target_dir: Path | None = None, verbose=True
 ) -> None:

pyodide_build/out_of_tree/build.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from textwrap import dedent
 
 from build import ConfigSettingsType
 
@@ -8,11 +9,40 @@
 from pyodide_build.spec import _BuildSpecExports
 
 
+def _create_ignore_files(directory: Path) -> None:
+    directory.joinpath(".gitignore").write_text(
+        dedent("""\
+        # Created by pyodide-build
+        *
+    """),
+        encoding="utf-8",
+    )
+
+    directory.joinpath(".hgignore").write_text(
+        dedent("""\
+        # Created by pyodide-build
+        syntax: glob
+        **/*
+    """),
+        encoding="utf-8",
+    )
+
+
+def _prepare_build_dir(build_dir: Path) -> None:
+    # create a persistent build dir in the source dir
+    build_dir.mkdir(exist_ok=True)
+    # don't track the build dir in version control,
+    # helps if building in a git/mercurial repo
+    _create_ignore_files(build_dir)
+
+
 def run(
     srcdir: Path,
     outdir: Path,
     exports: _BuildSpecExports,
     config_settings: ConfigSettingsType,
+    isolation: bool = True,
+    skip_dependency_check: bool = False,
 ) -> Path:
     outdir = outdir.resolve()
     cflags = build_env.get_build_flag("SIDE_MODULE_CFLAGS")
@@ -28,6 +58,9 @@ def run(
     env = os.environ.copy()
     env.update(build_env.get_build_environment_vars(get_pyodide_root()))
 
+    build_dir = srcdir / ".pyodide_build"
+    _prepare_build_dir(build_dir)
+
     build_env_ctx = pypabuild.get_build_env(
         env=env,
         pkgname="",
@@ -36,10 +69,18 @@ def run(
         ldflags=ldflags,
         target_install_dir=target_install_dir,
         exports=exports,
+        build_dir=build_dir,
     )
 
     with build_env_ctx as env:
-        built_wheel = pypabuild.build(srcdir, outdir, env, config_settings)
+        built_wheel = pypabuild.build(
+            srcdir,
+            outdir,
+            env,
+            config_settings,
+            isolation=isolation,
+            skip_dependency_check=skip_dependency_check,
+        )
 
     wheel_path = Path(built_wheel)
     if "emscripten" in wheel_path.name: