Merge pull request #770 from webknjaz/packaging/fix-build-hang-on-spawn-in-cythonize

📦 Disable `cythonize` parallelism w/ `spawn`

Author: webknjaz

docs/changelog-fragments/762.packaging.rst

@@ -0,0 +1 @@
+770.packaging.rst

docs/changelog-fragments/769.packaging.rst

@@ -0,0 +1 @@
+770.packaging.rst

docs/changelog-fragments/770.packaging.rst

@@ -0,0 +1,7 @@
+A workaround has been applied to the in-tree build backend that prevents
+Cython from hanging when ``libssh`` header files are missing
+-- by :user:`webknjaz`.
+
+The patch makes ``cythonize()`` single-threaded because :mod:`multiprocessing`
+gets stuck. The upstream will eventually fix this by migrating to
+:mod:`concurrent.futures`.

packaging/pep517_backend/_backend.py

@@ -54,7 +54,7 @@
 
 from ._compat import chdir_cm, nullcontext_cm  # noqa: WPS436
 from ._cython_configuration import (  # noqa: WPS436
-    get_local_cython_config as _get_local_cython_config,
+    get_local_cythonize_config as _get_local_cython_config,
 )
 from ._cython_configuration import (  # noqa: WPS436
     make_cythonize_cli_args_from_config as _make_cythonize_cli_args_from_config,

packaging/pep517_backend/_cython_configuration.py

@@ -21,6 +21,63 @@ def get_local_cython_config() -> dict:
     :returns: config section from ``pyproject.toml``
     :rtype: dict
 
+    This basically reads entries from::
+
+        [tool.local.cython]
+        # Env vars provisioned during cythonize call
+        src = ["src/**/*.pyx"]
+
+        [tool.local.cython.env]
+        # Env vars provisioned during cythonize call
+        LDFLAGS = "-lssh"
+
+        [tool.local.cython.flags]
+        # This section can contain the following booleans:
+        # * annotate — generate annotated HTML page for source files
+        # * build — build extension modules using distutils
+        # * inplace — build extension modules in place using distutils (implies -b)
+        # * force — force recompilation
+        # * quiet — be less verbose during compilation
+        # * lenient — increase Python compat by ignoring some compile time errors
+        # * keep-going — compile as much as possible, ignore compilation failures
+        annotate = false
+        build = false
+        inplace = true
+        force = true
+        quiet = false
+        lenient = false
+        keep-going = false
+
+        [tool.local.cython.kwargs]
+        # This section can contain args that have values:
+        # * exclude=PATTERN      exclude certain file patterns from the compilation
+        # * parallel=N    run builds in N parallel jobs (default: calculated per system)
+        exclude = "**.py"
+        parallel = 12
+
+        [tool.local.cython.kwargs.directives]
+        # This section can contain compiler directives
+        # NAME = "VALUE"
+
+        [tool.local.cython.kwargs.compile-time-env]
+        # This section can contain compile time env vars
+        # NAME = "VALUE"
+
+        [tool.local.cython.kwargs.options]
+        # This section can contain cythonize options
+        # NAME = "VALUE"
+    """
+    config_toml_txt = (Path.cwd().resolve() / 'pyproject.toml').read_text()
+    config_mapping = load_toml_from_string(config_toml_txt)
+    return config_mapping['tool']['local']['cython']
+
+
+def get_local_cythonize_config() -> dict:
+    """Grab optional build dependencies from pyproject.toml config.
+
+    :returns: config section from ``pyproject.toml``
+    :rtype: dict
+
     This basically reads entries from::
 
         [tool.local.cythonize]

pyproject.toml

@@ -15,6 +15,60 @@ requires = [
 backend-path = ["packaging"]  # requires 'Pip>=20' or 'pep517>=0.6.0'
 build-backend = "pep517_backend.hooks"
 
+[tool.local.cython]
+# This attr can contain multiple globs
+src = ["src/**/*.pyx"]
+
+[tool.local.cython.env]
+# Env vars provisioned during cythonize call
+#ANSIBLE_PYLIBSSH_CYTHON_TRACING = "1"
+#CFLAGS = "-DCYTHON_TRACE=1 ${CFLAGS}"
+LDFLAGS = "-lssh ${LDFLAGS}"
+
+[tool.local.cython.flags]
+# This section can contain the following booleans:
+# * annotate — generate annotated HTML page for source files
+# * build — build extension modules using distutils
+# * inplace — build extension modules in place using distutils (implies -b)
+# * force — force recompilation
+# * quiet — be less verbose during compilation
+# * lenient — increase Python compat by ignoring some compile time errors
+# * keep-going — compile as much as possible, ignore compilation failures
+annotate = false
+build = false
+inplace = true
+force = true
+quiet = false
+lenient = false
+keep-going = false
+
+[tool.local.cython.kwargs]
+# This section can contain args tha have values:
+# * exclude=PATTERN      exclude certain file patterns from the compilation
+# * parallel=N    run builds in N parallel jobs (default: calculated per system)
+# exclude = "**.py"
+# parallel = 12
+
+[tool.local.cython.kwargs.directive]
+# This section can contain compiler directives
+# Ref: https://github.com/cython/cython/blob/d6e6de9/Cython/Compiler/Options.py#L170-L242
+embedsignature = "True"
+emit_code_comments = "True"
+linetrace = "True"
+profile = "True"
+
+[tool.local.cython.kwargs.compile-time-env]
+# This section can contain compile time env vars
+
+[tool.local.cython.kwargs.option]
+# This section can contain cythonize options
+# Ref: https://github.com/cython/cython/blob/d6e6de9/Cython/Compiler/Options.py#L694-L730
+#docstrings = "True"
+#embed_pos_in_docstring = "True"
+#warning_errors = "True"
+#error_on_unknown_names = "True"
+#error_on_uninitialized = "True"
+
 [tool.local.cythonize]
 # This attr can contain multiple globs
 src = ["src/**/*.pyx"]
@@ -47,7 +101,13 @@ keep-going = false
 # * exclude=PATTERN      exclude certain file patterns from the compilation
 # * parallel=N    run builds in N parallel jobs (default: calculated per system)
 # exclude = "**.py"
-# parallel = 12
+#
+# NOTE: Setting parallelism to 1 works around a bug in the ``cythonize``
+# NOTE: script that hangs on macOS under Python 3.8+ due to the use of
+# NOTE: the ``spawn`` method in ``multiprocessing``. This patch can be
+# NOTE: reverted once the fix PR is usable upstream.
+# Ref: https://github.com/cython/cython/pull/7183
+parallel = 1
 
 [tool.local.cythonize.kwargs.directive]
 # This section can contain compiler directives