diff --git a/.github/workflows/nox.yml b/.github/workflows/nox.yml index a03b2d7b..69dc0051 100644 --- a/.github/workflows/nox.yml +++ b/.github/workflows/nox.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 339c9db0..dc5fb607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 🗞️ Changelog +## [Unreleased] + +**New features:** + +- Use `InterpreterPoolExecutor` as default executor on Python 3.14+ [\#488](https://github.com/python-adaptive/adaptive/pull/488). See also [Python documentation](https://docs.python.org/3.14/whatsnew/3.14.html#pep-734-multiple-interpreters-in-the-standard-library). + ## [v1.4.0](https://github.com/python-adaptive/adaptive/tree/v1.3.0) (2025-05-13) [Full Changelog](https://github.com/python-adaptive/adaptive/compare/v1.3.2...v.1.4.0) diff --git a/adaptive/_types.py b/adaptive/_types.py index 3f56bf59..540b28f0 100644 --- a/adaptive/_types.py +++ b/adaptive/_types.py @@ -2,7 +2,8 @@ # Workaround described in https://github.com/agronholm/typeguard/issues/456 import concurrent.futures as concurrent -from typing import TypeAlias +import sys +from typing import TYPE_CHECKING, TypeAlias import distributed import ipyparallel @@ -11,14 +12,31 @@ from adaptive.utils import SequentialExecutor -ExecutorTypes: TypeAlias = ( - concurrent.ProcessPoolExecutor - | concurrent.ThreadPoolExecutor - | SequentialExecutor - | loky.reusable_executor._ReusablePoolExecutor - | distributed.Client - | distributed.cfexecutor.ClientExecutor - | mpi4py.futures.MPIPoolExecutor - | ipyparallel.Client - | ipyparallel.client.view.ViewExecutor -) +# For Python 3.14+, include InterpreterPoolExecutor in the type alias +if sys.version_info >= (3, 14): + if TYPE_CHECKING: + # Type checkers will see this when checking Python 3.14+ code + ExecutorTypes: TypeAlias = ( + concurrent.ProcessPoolExecutor + | concurrent.ThreadPoolExecutor + | concurrent.InterpreterPoolExecutor # type: ignore[attr-defined] + | SequentialExecutor + | loky.reusable_executor._ReusablePoolExecutor + | distributed.Client + | distributed.cfexecutor.ClientExecutor + | mpi4py.futures.MPIPoolExecutor + | ipyparallel.Client + | ipyparallel.client.view.ViewExecutor + ) +else: + ExecutorTypes: TypeAlias = ( + concurrent.ProcessPoolExecutor + | concurrent.ThreadPoolExecutor + | SequentialExecutor + | loky.reusable_executor._ReusablePoolExecutor + | distributed.Client + | distributed.cfexecutor.ClientExecutor + | mpi4py.futures.MPIPoolExecutor + | ipyparallel.Client + | ipyparallel.client.view.ViewExecutor + ) diff --git a/adaptive/runner.py b/adaptive/runner.py index b2e5ec9a..a1f08d4a 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -8,6 +8,7 @@ import itertools import pickle import platform +import sys import time import traceback import warnings @@ -44,7 +45,13 @@ # -- Runner definitions -if platform.system() == "Linux": +_has_interpreter_pool = sys.version_info >= (3, 14) and hasattr( + concurrent, "InterpreterPoolExecutor" +) + +if _has_interpreter_pool: + _default_executor = concurrent.InterpreterPoolExecutor # type: ignore[misc,attr-defined] +elif platform.system() == "Linux": _default_executor = concurrent.ProcessPoolExecutor # type: ignore[misc] else: # On Windows and MacOS functions, the __main__ module must be @@ -86,8 +93,10 @@ class BaseRunner(metaclass=abc.ABCMeta): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` + on Python 3.14+, a `~concurrent.futures.ProcessPoolExecutor` on + Python < 3.14 on Linux, and a `loky.get_reusable_executor` on + Python < 3.14 on MacOS and Windows. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -373,8 +382,10 @@ class BlockingRunner(BaseRunner): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` + on Python 3.14+, a `~concurrent.futures.ProcessPoolExecutor` on + Python < 3.14 on Linux, and a `loky.get_reusable_executor` on + Python < 3.14 on MacOS and Windows. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -520,8 +531,10 @@ class AsyncRunner(BaseRunner): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` + on Python 3.14+, a new `~concurrent.futures.ProcessPoolExecutor` on + Python < 3.14 on Linux, and a `loky.get_reusable_executor` on + Python < 3.14 on MacOS and Windows. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -1023,6 +1036,8 @@ def _get_ncores( import mpi4py.futures if with_ipyparallel and isinstance(ex, ipyparallel.client.view.ViewExecutor): return len(ex.view) + elif _has_interpreter_pool and isinstance(ex, concurrent.InterpreterPoolExecutor): # type: ignore[attr-defined] + return ex._max_workers # type: ignore[union-attr] elif isinstance(ex, concurrent.ProcessPoolExecutor | concurrent.ThreadPoolExecutor): return ex._max_workers # type: ignore[union-attr] elif isinstance(ex, loky.reusable_executor._ReusablePoolExecutor): diff --git a/adaptive/tests/test_runner.py b/adaptive/tests/test_runner.py index 0bb68c59..5bb4e644 100644 --- a/adaptive/tests/test_runner.py +++ b/adaptive/tests/test_runner.py @@ -263,3 +263,26 @@ def counting_ask(self, n, tell_pending=True): finally: # Restore original method Learner1D.ask = original_ask + + +def test_interpreter_pool_executor_detection(): + """Test that InterpreterPoolExecutor is detected and used in Python 3.14+.""" + import concurrent.futures as concurrent + import sys + + from adaptive.runner import _default_executor, _has_interpreter_pool + + if sys.version_info >= (3, 14) and hasattr(concurrent, "InterpreterPoolExecutor"): + # On Python 3.14+, InterpreterPoolExecutor should be available and used + assert _has_interpreter_pool is True + assert _default_executor == concurrent.InterpreterPoolExecutor + else: + # On older Python versions, it should not be available + assert _has_interpreter_pool is False + # Should fall back to ProcessPoolExecutor on Linux or loky on others + if OPERATING_SYSTEM == "Linux": + assert _default_executor == concurrent.ProcessPoolExecutor + else: + import loky + + assert _default_executor == loky.get_reusable_executor diff --git a/docs/source/tutorial/tutorial.parallelism.md b/docs/source/tutorial/tutorial.parallelism.md index 5decc61d..7d1a5221 100644 --- a/docs/source/tutorial/tutorial.parallelism.md +++ b/docs/source/tutorial/tutorial.parallelism.md @@ -16,7 +16,13 @@ Often you will want to evaluate the function on some remote computing resources. ## `concurrent.futures` -On Unix-like systems by default {class}`adaptive.Runner` creates a {class}`~concurrent.futures.ProcessPoolExecutor`, but you can also pass one explicitly e.g. to limit the number of workers: +By default, {class}`adaptive.Runner` automatically selects the best executor based on your Python version and platform: + +- **Python 3.14+**: Uses {class}`~concurrent.futures.InterpreterPoolExecutor` (provides better isolation and performance) +- **Python < 3.14 on Unix-like systems**: Uses {class}`~concurrent.futures.ProcessPoolExecutor` +- **Python < 3.14 on Windows/macOS**: Uses `loky.get_reusable_executor` (better compatibility in interactive environments) + +You can also pass an executor explicitly, e.g. to limit the number of workers: ```python from concurrent.futures import ProcessPoolExecutor diff --git a/noxfile.py b/noxfile.py index 71a2217a..06b816fc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,7 +6,7 @@ nox.options.default_venv_backend = "uv" -python = ["3.11", "3.12", "3.13"] +python = ["3.11", "3.12", "3.13", "3.14"] num_cpus = os.cpu_count() or 1 xdist = ("-n", "auto") if num_cpus > 2 else () diff --git a/pyproject.toml b/pyproject.toml index 5c7186e9..e3f4d28f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dependencies = [ "scipy",