From b186cd331bafbf058e552577ba01a8bdc17ce676 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Oct 2025 09:34:01 +0000 Subject: [PATCH 1/9] Initial plan From bd1a8df9d8dda5a8e07aaa6f44dfabe2806e718c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Oct 2025 09:43:36 +0000 Subject: [PATCH 2/9] Add InterpreterPoolExecutor support for Python 3.14+ Co-authored-by: akhmerov <2069677+akhmerov@users.noreply.github.com> --- adaptive/_types.py | 42 +++++++++++++++++++++++++---------- adaptive/runner.py | 11 ++++++++- adaptive/tests/test_runner.py | 24 ++++++++++++++++++++ 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/adaptive/_types.py b/adaptive/_types.py index 3f56bf59..540b28f0 100644 --- a/adaptive/_types.py +++ b/adaptive/_types.py @@ -2,7 +2,8 @@ # Workaround described in https://github.com/agronholm/typeguard/issues/456 import concurrent.futures as concurrent -from typing import TypeAlias +import sys +from typing import TYPE_CHECKING, TypeAlias import distributed import ipyparallel @@ -11,14 +12,31 @@ from adaptive.utils import SequentialExecutor -ExecutorTypes: TypeAlias = ( - concurrent.ProcessPoolExecutor - | concurrent.ThreadPoolExecutor - | SequentialExecutor - | loky.reusable_executor._ReusablePoolExecutor - | distributed.Client - | distributed.cfexecutor.ClientExecutor - | mpi4py.futures.MPIPoolExecutor - | ipyparallel.Client - | ipyparallel.client.view.ViewExecutor -) +# For Python 3.14+, include InterpreterPoolExecutor in the type alias +if sys.version_info >= (3, 14): + if TYPE_CHECKING: + # Type checkers will see this when checking Python 3.14+ code + ExecutorTypes: TypeAlias = ( + concurrent.ProcessPoolExecutor + | concurrent.ThreadPoolExecutor + | concurrent.InterpreterPoolExecutor # type: ignore[attr-defined] + | SequentialExecutor + | loky.reusable_executor._ReusablePoolExecutor + | distributed.Client + | distributed.cfexecutor.ClientExecutor + | mpi4py.futures.MPIPoolExecutor + | ipyparallel.Client + | ipyparallel.client.view.ViewExecutor + ) +else: + ExecutorTypes: TypeAlias = ( + concurrent.ProcessPoolExecutor + | concurrent.ThreadPoolExecutor + | SequentialExecutor + | loky.reusable_executor._ReusablePoolExecutor + | distributed.Client + | distributed.cfexecutor.ClientExecutor + | mpi4py.futures.MPIPoolExecutor + | ipyparallel.Client + | ipyparallel.client.view.ViewExecutor + ) diff --git a/adaptive/runner.py b/adaptive/runner.py index b2e5ec9a..8ef27d73 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -44,7 +44,14 @@ # -- Runner definitions -if platform.system() == "Linux": +# Check if InterpreterPoolExecutor is available (Python 3.14+) +_has_interpreter_pool = hasattr(concurrent, "InterpreterPoolExecutor") + +if _has_interpreter_pool: + # Use InterpreterPoolExecutor for Python 3.14+ + # It provides better isolation and performance than ProcessPoolExecutor + _default_executor = concurrent.InterpreterPoolExecutor # type: ignore[misc,attr-defined] +elif platform.system() == "Linux": _default_executor = concurrent.ProcessPoolExecutor # type: ignore[misc] else: # On Windows and MacOS functions, the __main__ module must be @@ -1025,6 +1032,8 @@ def _get_ncores( return len(ex.view) elif isinstance(ex, concurrent.ProcessPoolExecutor | concurrent.ThreadPoolExecutor): return ex._max_workers # type: ignore[union-attr] + elif _has_interpreter_pool and isinstance(ex, concurrent.InterpreterPoolExecutor): # type: ignore[attr-defined] + return ex._max_workers # type: ignore[union-attr] elif isinstance(ex, loky.reusable_executor._ReusablePoolExecutor): return ex._max_workers # type: ignore[union-attr] elif isinstance(ex, SequentialExecutor): diff --git a/adaptive/tests/test_runner.py b/adaptive/tests/test_runner.py index 0bb68c59..fd52e7a1 100644 --- a/adaptive/tests/test_runner.py +++ b/adaptive/tests/test_runner.py @@ -263,3 +263,27 @@ def counting_ask(self, n, tell_pending=True): finally: # Restore original method Learner1D.ask = original_ask + + +def test_interpreter_pool_executor_detection(): + """Test that InterpreterPoolExecutor is detected and used in Python 3.14+.""" + import concurrent.futures as concurrent + import sys + + from adaptive.runner import _default_executor, _has_interpreter_pool + + if sys.version_info >= (3, 14) and hasattr(concurrent, "InterpreterPoolExecutor"): + # On Python 3.14+, InterpreterPoolExecutor should be available and used + assert _has_interpreter_pool is True + assert _default_executor == concurrent.InterpreterPoolExecutor + else: + # On older Python versions, it should not be available + assert _has_interpreter_pool is False + # Should fall back to ProcessPoolExecutor on Linux or loky on others + if OPERATING_SYSTEM == "Linux": + assert _default_executor == concurrent.ProcessPoolExecutor + else: + import loky + + assert _default_executor == loky.get_reusable_executor + From cbc34129b15180be34626688790f6c9b6b953424 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Oct 2025 09:48:43 +0000 Subject: [PATCH 3/9] Update documentation and changelog for InterpreterPoolExecutor Co-authored-by: akhmerov <2069677+akhmerov@users.noreply.github.com> --- CHANGELOG.md | 10 ++++++++++ adaptive/runner.py | 18 ++++++++++++------ docs/source/tutorial/tutorial.parallelism.md | 8 +++++++- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 339c9db0..dae01577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # 🗞️ Changelog +## [Unreleased] + +**New features:** + +- Use `InterpreterPoolExecutor` as default executor on Python 3.14+ [\#XXX](https://github.com/python-adaptive/adaptive/pull/XXX) + - Automatically detects and uses `InterpreterPoolExecutor` when available (Python 3.14+) + - Provides better isolation and performance than `ProcessPoolExecutor` + - Falls back to `ProcessPoolExecutor` on Linux or `loky` on Windows/macOS for older Python versions + - Maintains full backward compatibility with existing code + ## [v1.4.0](https://github.com/python-adaptive/adaptive/tree/v1.3.0) (2025-05-13) [Full Changelog](https://github.com/python-adaptive/adaptive/compare/v1.3.2...v.1.4.0) diff --git a/adaptive/runner.py b/adaptive/runner.py index 8ef27d73..316f9857 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -93,8 +93,10 @@ class BaseRunner(metaclass=abc.ABCMeta): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` + on Python 3.14+, a new `~concurrent.futures.ProcessPoolExecutor` on + Python < 3.14 on Linux, and a `loky.get_reusable_executor` on + Python < 3.14 on MacOS and Windows. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -380,8 +382,10 @@ class BlockingRunner(BaseRunner): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` + on Python 3.14+, a new `~concurrent.futures.ProcessPoolExecutor` on + Python < 3.14 on Linux, and a `loky.get_reusable_executor` on + Python < 3.14 on MacOS and Windows. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. @@ -527,8 +531,10 @@ class AsyncRunner(BaseRunner): `mpi4py.futures.MPIPoolExecutor`, `ipyparallel.Client` or\ `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. - If not provided, a new `~concurrent.futures.ProcessPoolExecutor` on - Linux, and a `loky.get_reusable_executor` on MacOS and Windows. + If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` + on Python 3.14+, a new `~concurrent.futures.ProcessPoolExecutor` on + Python < 3.14 on Linux, and a `loky.get_reusable_executor` on + Python < 3.14 on MacOS and Windows. ntasks : int, optional The number of concurrent function evaluations. Defaults to the number of cores available in `executor`. diff --git a/docs/source/tutorial/tutorial.parallelism.md b/docs/source/tutorial/tutorial.parallelism.md index 5decc61d..7d1a5221 100644 --- a/docs/source/tutorial/tutorial.parallelism.md +++ b/docs/source/tutorial/tutorial.parallelism.md @@ -16,7 +16,13 @@ Often you will want to evaluate the function on some remote computing resources. ## `concurrent.futures` -On Unix-like systems by default {class}`adaptive.Runner` creates a {class}`~concurrent.futures.ProcessPoolExecutor`, but you can also pass one explicitly e.g. to limit the number of workers: +By default, {class}`adaptive.Runner` automatically selects the best executor based on your Python version and platform: + +- **Python 3.14+**: Uses {class}`~concurrent.futures.InterpreterPoolExecutor` (provides better isolation and performance) +- **Python < 3.14 on Unix-like systems**: Uses {class}`~concurrent.futures.ProcessPoolExecutor` +- **Python < 3.14 on Windows/macOS**: Uses `loky.get_reusable_executor` (better compatibility in interactive environments) + +You can also pass an executor explicitly, e.g. to limit the number of workers: ```python from concurrent.futures import ProcessPoolExecutor From b0875bcb99807655862efbf9dd52d3706a530bd4 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Thu, 9 Oct 2025 12:09:42 +0200 Subject: [PATCH 4/9] precommit --- adaptive/tests/test_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/adaptive/tests/test_runner.py b/adaptive/tests/test_runner.py index fd52e7a1..5bb4e644 100644 --- a/adaptive/tests/test_runner.py +++ b/adaptive/tests/test_runner.py @@ -286,4 +286,3 @@ def test_interpreter_pool_executor_detection(): import loky assert _default_executor == loky.get_reusable_executor - From 3fff71bae2acfda24881b62a0df148eb684661a0 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Thu, 9 Oct 2025 12:31:52 +0200 Subject: [PATCH 5/9] shorten the changelog --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dae01577..dc5fb607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,7 @@ **New features:** -- Use `InterpreterPoolExecutor` as default executor on Python 3.14+ [\#XXX](https://github.com/python-adaptive/adaptive/pull/XXX) - - Automatically detects and uses `InterpreterPoolExecutor` when available (Python 3.14+) - - Provides better isolation and performance than `ProcessPoolExecutor` - - Falls back to `ProcessPoolExecutor` on Linux or `loky` on Windows/macOS for older Python versions - - Maintains full backward compatibility with existing code +- Use `InterpreterPoolExecutor` as default executor on Python 3.14+ [\#488](https://github.com/python-adaptive/adaptive/pull/488). See also [Python documentation](https://docs.python.org/3.14/whatsnew/3.14.html#pep-734-multiple-interpreters-in-the-standard-library). ## [v1.4.0](https://github.com/python-adaptive/adaptive/tree/v1.3.0) (2025-05-13) From 1a81fea555b6feaf5238c433c52330a8c7e01e19 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Thu, 9 Oct 2025 12:32:26 +0200 Subject: [PATCH 6/9] support and test python 3.14 --- .github/workflows/nox.yml | 2 +- noxfile.py | 2 +- pyproject.toml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nox.yml b/.github/workflows/nox.yml index a03b2d7b..69dc0051 100644 --- a/.github/workflows/nox.yml +++ b/.github/workflows/nox.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/noxfile.py b/noxfile.py index 71a2217a..06b816fc 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,7 +6,7 @@ nox.options.default_venv_backend = "uv" -python = ["3.11", "3.12", "3.13"] +python = ["3.11", "3.12", "3.13", "3.14"] num_cpus = os.cpu_count() or 1 xdist = ("-n", "auto") if num_cpus > 2 else () diff --git a/pyproject.toml b/pyproject.toml index 5c7186e9..e3f4d28f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dependencies = [ "scipy", From a474e29dee84cb476c07146bb5d0b9477173d915 Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Thu, 9 Oct 2025 12:42:58 +0200 Subject: [PATCH 7/9] homogenize the check --- adaptive/runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/adaptive/runner.py b/adaptive/runner.py index 316f9857..e1ae3be4 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -8,6 +8,7 @@ import itertools import pickle import platform +import sys import time import traceback import warnings @@ -44,12 +45,9 @@ # -- Runner definitions -# Check if InterpreterPoolExecutor is available (Python 3.14+) -_has_interpreter_pool = hasattr(concurrent, "InterpreterPoolExecutor") +_has_interpreter_pool = sys.version_info >= (3, 14) if _has_interpreter_pool: - # Use InterpreterPoolExecutor for Python 3.14+ - # It provides better isolation and performance than ProcessPoolExecutor _default_executor = concurrent.InterpreterPoolExecutor # type: ignore[misc,attr-defined] elif platform.system() == "Linux": _default_executor = concurrent.ProcessPoolExecutor # type: ignore[misc] From 522babc3b6a348286152e035bb0d7cf3929c84dd Mon Sep 17 00:00:00 2001 From: Anton Akhmerov Date: Thu, 9 Oct 2025 12:44:58 +0200 Subject: [PATCH 8/9] mark ProcessPoolExecutor as not new --- adaptive/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adaptive/runner.py b/adaptive/runner.py index e1ae3be4..272125e6 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -92,7 +92,7 @@ class BaseRunner(metaclass=abc.ABCMeta): `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` - on Python 3.14+, a new `~concurrent.futures.ProcessPoolExecutor` on + on Python 3.14+, a `~concurrent.futures.ProcessPoolExecutor` on Python < 3.14 on Linux, and a `loky.get_reusable_executor` on Python < 3.14 on MacOS and Windows. ntasks : int, optional @@ -381,7 +381,7 @@ class BlockingRunner(BaseRunner): `loky.get_reusable_executor`, optional The executor in which to evaluate the function to be learned. If not provided, a new `~concurrent.futures.InterpreterPoolExecutor` - on Python 3.14+, a new `~concurrent.futures.ProcessPoolExecutor` on + on Python 3.14+, a `~concurrent.futures.ProcessPoolExecutor` on Python < 3.14 on Linux, and a `loky.get_reusable_executor` on Python < 3.14 on MacOS and Windows. ntasks : int, optional From 00589793520d0284b166c3a6ba5ce0a30a9205c4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 9 Oct 2025 11:00:11 +0000 Subject: [PATCH 9/9] Fix Python 3.14 test failures: check hasattr and reorder isinstance checks Co-authored-by: akhmerov <2069677+akhmerov@users.noreply.github.com> --- adaptive/runner.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/adaptive/runner.py b/adaptive/runner.py index 272125e6..a1f08d4a 100644 --- a/adaptive/runner.py +++ b/adaptive/runner.py @@ -45,7 +45,9 @@ # -- Runner definitions -_has_interpreter_pool = sys.version_info >= (3, 14) +_has_interpreter_pool = sys.version_info >= (3, 14) and hasattr( + concurrent, "InterpreterPoolExecutor" +) if _has_interpreter_pool: _default_executor = concurrent.InterpreterPoolExecutor # type: ignore[misc,attr-defined] @@ -1034,10 +1036,10 @@ def _get_ncores( import mpi4py.futures if with_ipyparallel and isinstance(ex, ipyparallel.client.view.ViewExecutor): return len(ex.view) - elif isinstance(ex, concurrent.ProcessPoolExecutor | concurrent.ThreadPoolExecutor): - return ex._max_workers # type: ignore[union-attr] elif _has_interpreter_pool and isinstance(ex, concurrent.InterpreterPoolExecutor): # type: ignore[attr-defined] return ex._max_workers # type: ignore[union-attr] + elif isinstance(ex, concurrent.ProcessPoolExecutor | concurrent.ThreadPoolExecutor): + return ex._max_workers # type: ignore[union-attr] elif isinstance(ex, loky.reusable_executor._ReusablePoolExecutor): return ex._max_workers # type: ignore[union-attr] elif isinstance(ex, SequentialExecutor):