Start kdist workers with spawn (#4601)

* Ensure that the start method is the same on all platforms
(https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods)
* Set the log level in the worker. This fixes an issue on macOS where
the worker logs are not visible.

Author: tothtamas28

pyk/src/pyk/kdist/__main__.py

@@ -9,23 +9,24 @@
 from pyk.cli.utils import loglevel
 
 from ..kdist import kdist, target_ids
+from .utils import LOG_FORMAT
 
 if TYPE_CHECKING:
     from argparse import Namespace
     from typing import Final
 
 
 _LOGGER: Final = logging.getLogger(__name__)
-_LOG_FORMAT: Final = '%(levelname)s %(asctime)s %(name)s - %(message)s'
 
 
 def main() -> None:
     args = _parse_arguments()
+    log_level = loglevel(args)
 
-    logging.basicConfig(level=loglevel(args), format=_LOG_FORMAT)
+    logging.basicConfig(level=log_level, format=LOG_FORMAT)
 
     if args.command == 'build':
-        _exec_build(**vars(args))
+        _exec_build(log_level=log_level, **vars(args))
 
     elif args.command == 'clean':
         _exec_clean(args.target)
@@ -45,6 +46,7 @@ def _exec_build(
     targets: list[str],
     args: list[str],
     jobs: int,
+    log_level: int,
     force: bool,
     verbose: bool,
     debug: bool,
@@ -54,6 +56,7 @@ def _exec_build(
         target_ids=_process_targets(targets),
         args=_process_args(args),
         jobs=jobs,
+        log_level=log_level,
         force=force,
         verbose=verbose or debug,
         clean=clean,

pyk/src/pyk/kdist/_kdist.py

@@ -3,6 +3,7 @@
 import concurrent.futures
 import json
 import logging
+import multiprocessing
 import os
 import shutil
 from concurrent.futures import ProcessPoolExecutor
@@ -20,6 +21,7 @@
 from . import utils
 from ._cache import target_cache
 from .api import TargetId
+from .utils import LOG_FORMAT
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator, Mapping
@@ -84,6 +86,7 @@ def build(
         *,
         args: Mapping[str, str] | None = None,
         jobs: int = 1,
+        log_level: int = logging.WARNING,
         force: bool = False,
         verbose: bool = False,
         clean: bool = True,
@@ -99,14 +102,15 @@ def build(
         deps_fqns = [target_id.full_name for target_id in dep_ids]
         _LOGGER.info(f"Building targets: {', '.join(deps_fqns)}")
 
-        with ProcessPoolExecutor(max_workers=jobs) as pool:
+        with ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
             pending: dict[Future[Path], TargetId] = {}
 
             def submit(target_id: TargetId) -> None:
                 future = pool.submit(
                     self._build_target,
                     target_id=target_id,
                     args=args,
+                    log_level=log_level,
                     force=force,
                     verbose=verbose,
                     clean=clean,
@@ -134,10 +138,13 @@ def _build_target(
         target_id: TargetId,
         args: dict[str, Any],
         *,
+        log_level: int,
         force: bool,
         verbose: bool,
         clean: bool,
     ) -> Path:
+        logging.basicConfig(level=log_level, format=LOG_FORMAT)
+
         target = target_cache().resolve(target_id)
         output_dir = self._target_dir(target_id)
         manifest_file = self._manifest_file(target_id)

pyk/src/pyk/kdist/utils.py

@@ -10,7 +10,10 @@
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
-    from typing import Any
+    from typing import Any, Final
+
+
+LOG_FORMAT: Final = '%(levelname)s %(asctime)s %(name)s - %(message)s'
 
 
 def package_path(obj: Any) -> Path: