From 0c858f9b7cc8c2a011d699aa0f6e506a6d7f7824 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 29 Oct 2025 11:32:14 +0800 Subject: [PATCH 1/8] fix: Add __mp_main__ as a duplicate for __main__ for pickle to work Signed-off-by: yihong0618 --- Lib/profiling/sampling/_sync_coordinator.py | 11 ++++- .../test_profiling/test_sampling_profiler.py | 44 +++++++++++++++++++ ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 + 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 79e8858ca17529..b41fa3c4e279b8 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -10,6 +10,7 @@ import socket import runpy import time +import types from typing import List, NoReturn @@ -176,9 +177,15 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: with open(script_path, 'rb') as f: source_code = f.read() - # Compile and execute the script + # gh-140729: Create a __mp_main__ module to allow pickling + main_module = types.ModuleType("__main__") + main_module.__file__ = script_path + main_module.__builtins__ = __builtins__ + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + code = compile(source_code, script_path, 'exec') - exec(code, {'__name__': '__main__', '__file__': script_path}) + exec(code, main_module.__dict__) + except FileNotFoundError as e: raise TargetError(f"Script file not found: {script_path}") from e except PermissionError as e: diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 59bc18b9bcf14d..4ede72eecf7460 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -2991,5 +2991,49 @@ def test_parse_mode_function(self): profiling.sampling.sample._parse_mode("invalid") +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = ''' +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +''' + with tempfile.NamedTemporaryFile( + mode='w', suffix='.py', delete=False + ) as script_file: + script_file.write(test_script) + script_file.flush() + script_name = script_file.name + + result = subprocess.run( + [ + sys.executable, + "-m", "profiling.sampling.sample", + "-d", "1", + "-i", "100000", + script_name + ], + capture_output=True, + text=True, + timeout=10 + ) + + self.assertIn("Results: [2, 4, 6]", result.stdout) + self.assertNotIn("Can't pickle", result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst new file mode 100644 index 00000000000000..a7fef83628d0c9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst @@ -0,0 +1,2 @@ +Fix: Add __mp_main__ as a duplicate for __main__ for pickle to work in +sampling From 4cc048206c8a528b210f03758a41db2f6aa06037 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 29 Oct 2025 11:56:25 +0800 Subject: [PATCH 2/8] fix: skip the test if no permission Signed-off-by: yihong0618 --- Lib/test/test_profiling/test_sampling_profiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 4ede72eecf7460..40218ae6495910 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3031,6 +3031,9 @@ def worker(x): timeout=10 ) + if "PermissionError" in result.stderr: + self.skipTest("Insufficient permissions for remote profiling") + self.assertIn("Results: [2, 4, 6]", result.stdout) self.assertNotIn("Can't pickle", result.stderr) From 08de5c3d271d06cc54b0c651d7a35f3828a6a826 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 29 Oct 2025 12:53:52 +0800 Subject: [PATCH 3/8] fix: temp file leak Signed-off-by: yihong0618 --- Lib/test/test_profiling/test_sampling_profiler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 40218ae6495910..d28a8c5668d14e 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3018,13 +3018,15 @@ def worker(x): script_file.flush() script_name = script_file.name + self.addCleanup(os.unlink, script_name) + result = subprocess.run( [ sys.executable, "-m", "profiling.sampling.sample", "-d", "1", "-i", "100000", - script_name + script_name, ], capture_output=True, text=True, From e9bd308271015daaacefae79a11b5d0f65cd806c Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 29 Oct 2025 14:08:26 +0800 Subject: [PATCH 4/8] fix: make test better Signed-off-by: yihong0618 --- .../test_profiling/test_sampling_profiler.py | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index d28a8c5668d14e..276b6a11832aa6 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -22,7 +22,13 @@ from profiling.sampling.gecko_collector import GeckoCollector from test.support.os_helper import unlink -from test.support import force_not_colorized_test_class, SHORT_TIMEOUT +from test.support import ( + force_not_colorized_test_class, + SHORT_TIMEOUT, + script_helper, + os_helper, + SuppressCrashReport, +) from test.support.socket_helper import find_unused_port from test.support import requires_subprocess, is_emscripten from test.support import captured_stdout, captured_stderr @@ -3011,33 +3017,24 @@ def worker(x): results = list(executor.map(worker, [1, 2, 3])) print(f"Results: {results}") ''' - with tempfile.NamedTemporaryFile( - mode='w', suffix='.py', delete=False - ) as script_file: - script_file.write(test_script) - script_file.flush() - script_name = script_file.name - - self.addCleanup(os.unlink, script_name) - - result = subprocess.run( - [ - sys.executable, - "-m", "profiling.sampling.sample", - "-d", "1", - "-i", "100000", - script_name, - ], - capture_output=True, - text=True, - timeout=10 - ) + # Use test helpers to spawn a real Python subprocess so that + # PermissionError (if any) is emitted by the child on stderr and + # can be handled consistently with other tests. + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, 'test_process_pool_executor_pickle', test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python(script, stderr=subprocess.PIPE) as proc: + proc.wait() + stdout = proc.stdout.read() + stderr = proc.stderr.read() - if "PermissionError" in result.stderr: + if b"PermissionError" in stderr: self.skipTest("Insufficient permissions for remote profiling") - self.assertIn("Results: [2, 4, 6]", result.stdout) - self.assertNotIn("Can't pickle", result.stderr) + self.assertIn(b"Results: [2, 4, 6]", stdout) + self.assertNotIn(b"Can't pickle", stderr) if __name__ == "__main__": From 285a66f4d5c705cbc951ec67c75c804a867da846 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Wed, 29 Oct 2025 14:08:58 +0800 Subject: [PATCH 5/8] fix: drop comments Signed-off-by: yihong0618 --- Lib/test/test_profiling/test_sampling_profiler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 276b6a11832aa6..ac4018418853d4 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3017,9 +3017,6 @@ def worker(x): results = list(executor.map(worker, [1, 2, 3])) print(f"Results: {results}") ''' - # Use test helpers to spawn a real Python subprocess so that - # PermissionError (if any) is emitted by the child on stderr and - # can be handled consistently with other tests. with os_helper.temp_dir() as temp_dir: script = script_helper.make_script( temp_dir, 'test_process_pool_executor_pickle', test_script From 8b93ec3e2f9be551a7026c068da2e6e5a7d2f537 Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 2 Nov 2025 06:42:33 +0800 Subject: [PATCH 6/8] fix: apply suggestions Signed-off-by: yihong0618 --- .../test_profiling/test_sampling_profiler.py | 17 ++++++++++++----- ...25-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index f10afd1d068feb..25da94bb574235 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3038,16 +3038,23 @@ def worker(x): temp_dir, 'test_process_pool_executor_pickle', test_script ) with SuppressCrashReport(): - with script_helper.spawn_python(script, stderr=subprocess.PIPE) as proc: - proc.wait() + with script_helper.spawn_python( + "-m", "profiling.sampling.sample", + "-d", "1", + "-i", "100000", + script, + stderr=subprocess.PIPE, + text=True + ) as proc: + proc.wait(timeout=10) stdout = proc.stdout.read() stderr = proc.stderr.read() - if b"PermissionError" in stderr: + if "PermissionError" in stderr: self.skipTest("Insufficient permissions for remote profiling") - self.assertIn(b"Results: [2, 4, 6]", stdout) - self.assertNotIn(b"Can't pickle", stderr) + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst index a7fef83628d0c9..6725547667fb3c 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst @@ -1,2 +1,2 @@ -Fix: Add __mp_main__ as a duplicate for __main__ for pickle to work in -sampling +Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` +script can not be properly pickled and executed in worker processes. From c2e66dd2bff23de97394f1a7996bc89228bb622c Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 2 Nov 2025 07:32:11 +0800 Subject: [PATCH 7/8] fix: ensure flush that free thread on windows will not flush Signed-off-by: yihong0618 --- Lib/profiling/sampling/_sync_coordinator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 20b50ba9d9063d..ca8f0ee9366f3d 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -149,6 +149,9 @@ def _execute_module(module_name: str, module_args: List[str]) -> None: pass except Exception as e: raise TargetError(f"Error executing module '{module_name}': {e}") from e + finally: + sys.stdout.flush() + sys.stderr.flush() def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: @@ -198,6 +201,9 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: pass except Exception as e: raise TargetError(f"Error executing script '{script_path}': {e}") from e + finally: + sys.stdout.flush() + sys.stderr.flush() def main() -> NoReturn: From 59c1c1db58f7c3c5c2aa661f9da1ecc1e6340c9f Mon Sep 17 00:00:00 2001 From: yihong0618 Date: Sun, 2 Nov 2025 09:23:19 +0800 Subject: [PATCH 8/8] fix: apply suggestions and drop noise code Signed-off-by: yihong0618 --- Lib/profiling/sampling/_sync_coordinator.py | 6 ------ Lib/test/test_profiling/test_sampling_profiler.py | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index ca8f0ee9366f3d..20b50ba9d9063d 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -149,9 +149,6 @@ def _execute_module(module_name: str, module_args: List[str]) -> None: pass except Exception as e: raise TargetError(f"Error executing module '{module_name}': {e}") from e - finally: - sys.stdout.flush() - sys.stderr.flush() def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: @@ -201,9 +198,6 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: pass except Exception as e: raise TargetError(f"Error executing script '{script_path}': {e}") from e - finally: - sys.stdout.flush() - sys.stderr.flush() def main() -> NoReturn: diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 25da94bb574235..ad5bafaf7b82f4 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3040,13 +3040,13 @@ def worker(x): with SuppressCrashReport(): with script_helper.spawn_python( "-m", "profiling.sampling.sample", - "-d", "1", + "-d", "5", "-i", "100000", script, stderr=subprocess.PIPE, text=True ) as proc: - proc.wait(timeout=10) + proc.wait(timeout=SHORT_TIMEOUT) stdout = proc.stdout.read() stderr = proc.stderr.read()