From 7108bb91723a2d3e4c5f0e62fbf46afc030450de Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:56:12 +0800 Subject: [PATCH 1/9] =?UTF-8?q?Fix:=20Using=20`--import-mode=3Dimportlib`,?= =?UTF-8?q?=20a=20directory=20with=20the=20same=20name=20in=20the=20namesp?= =?UTF-8?q?ace=20package=20causes=20a=20KeyError.=EF=BC=88#12592=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- changelog/12592.bugfix.rst | 1 + src/_pytest/pathlib.py | 86 ++++++++++++++++++++++++-------------- testing/test_pathlib.py | 40 +++++++++++++++++- 3 files changed, 95 insertions(+), 32 deletions(-) create mode 100644 changelog/12592.bugfix.rst diff --git a/changelog/12592.bugfix.rst b/changelog/12592.bugfix.rst new file mode 100644 index 00000000000..74a4cb22011 --- /dev/null +++ b/changelog/12592.bugfix.rst @@ -0,0 +1 @@ +Fixed the issue of causes ``KeyError`` when using the parameter ``--import-mode=importlib`` in pytest>=8.2 . diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 81e52ea729d..0b3e861c707 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -10,6 +10,7 @@ import fnmatch from functools import partial from importlib.machinery import ModuleSpec +from importlib.machinery import PathFinder import importlib.util import itertools import os @@ -37,8 +38,12 @@ from _pytest.warning_types import PytestWarning -LOCK_TIMEOUT = 60 * 60 * 24 * 3 +if sys.version_info < (3, 11): + from importlib._bootstrap_external import _NamespaceLoader as NamespaceLoader +else: + from importlib.machinery import NamespaceLoader +LOCK_TIMEOUT = 60 * 60 * 24 * 3 _AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath) @@ -618,6 +623,34 @@ def _import_module_using_spec( If True, will call insert_missing_modules to create empty intermediate modules for made-up module names (when importing test files not reachable from sys.path). """ + # Attempt to import the parent module, seems is our responsibility: + # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311 + parent_module_name, _, name = module_name.rpartition(".") + parent_module: ModuleType | None = None + if parent_module_name: + parent_module = sys.modules.get(parent_module_name) + if parent_module is None: + if (module_path.parent / "__init__.py").is_file(): + parent_module_location = module_location + else: + parent_module_location = module_location.parent + + if module_path.name == "__init__.py": + parent_module_path = module_path.parent.parent + else: + parent_module_path = module_path.parent + + # Consider the parent module path as its __init__.py file, if it has one. + if (parent_module_path / "__init__.py").is_file(): + parent_module_path = parent_module_path / "__init__.py" + + parent_module = _import_module_using_spec( + parent_module_name, + parent_module_path, + parent_module_location, + insert_modules=insert_modules, + ) + # Checking with sys.meta_path first in case one of its hooks can import this module, # such as our own assertion-rewrite hook. for meta_importer in sys.meta_path: @@ -627,36 +660,18 @@ def _import_module_using_spec( if spec_matches_module_path(spec, module_path): break else: - spec = importlib.util.spec_from_file_location(module_name, str(module_path)) + loader = None + if "." in module_path.name and module_path.is_dir(): + # The `spec_from_file_location` matches a loader based on the file extension by default. + # For a namespace package, you need to manually specify a loader. + loader = NamespaceLoader(name, module_path, PathFinder()) + + spec = importlib.util.spec_from_file_location( + module_name, str(module_path), loader=loader + ) if spec_matches_module_path(spec, module_path): assert spec is not None - # Attempt to import the parent module, seems is our responsibility: - # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311 - parent_module_name, _, name = module_name.rpartition(".") - parent_module: ModuleType | None = None - if parent_module_name: - parent_module = sys.modules.get(parent_module_name) - if parent_module is None: - # Find the directory of this module's parent. - parent_dir = ( - module_path.parent.parent - if module_path.name == "__init__.py" - else module_path.parent - ) - # Consider the parent module path as its __init__.py file, if it has one. - parent_module_path = ( - parent_dir / "__init__.py" - if (parent_dir / "__init__.py").is_file() - else parent_dir - ) - parent_module = _import_module_using_spec( - parent_module_name, - parent_module_path, - parent_dir, - insert_modules=insert_modules, - ) - # Find spec and import this module. mod = importlib.util.module_from_spec(spec) sys.modules[module_name] = mod @@ -669,16 +684,25 @@ def _import_module_using_spec( if insert_modules: insert_missing_modules(sys.modules, module_name) return mod - return None def spec_matches_module_path(module_spec: ModuleSpec | None, module_path: Path) -> bool: """Return true if the given ModuleSpec can be used to import the given module path.""" - if module_spec is None or module_spec.origin is None: + if module_spec is None: return False - return Path(module_spec.origin) == module_path + if module_spec.origin: + return Path(module_spec.origin) == module_path + + # If this is a namespace package, compare the path with the `module_spec.submodule_Search_Locations` + # https://docs.python.org/zh-cn/3/library/importlib.html#importlib.machinery.ModuleSpec.submodule_search_locations + if module_spec.submodule_search_locations: + for _path in module_spec.submodule_search_locations: + if Path(_path) == module_path: + return True + + return False # Implement a special _is_same function on Windows which returns True if the two filenames diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 7c0a0e03d76..f907cd7310f 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -416,7 +416,7 @@ def test_no_meta_path_found( del sys.modules[module.__name__] monkeypatch.setattr( - importlib.util, "spec_from_file_location", lambda *args: None + importlib.util, "spec_from_file_location", lambda *args, **kwargs: None ) with pytest.raises(ImportError): import_path( @@ -1541,6 +1541,44 @@ def test_full_ns_packages_without_init_files( tmp_path / "src/dist2/ns/a/core/foo/m.py", consider_namespace_packages=True ) == (tmp_path / "src/dist2", "ns.a.core.foo.m") + def test_ns_multiple_levels_import_same_name_directory( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + ) -> None: + """Check KeyError with `--import-mode=importlib` (#12592).""" + self.setup_directories(tmp_path, monkeypatch, pytester) + code = dedent(""" + def test(): + assert "four lights" == "five lights" + """) + + # a subdirectories with the same name in the namespace package, + # along with a tests file + test_base_path = tmp_path / "src/dist2/com/company" + test_py = test_base_path / "a/b/c/test_demo.py" + test_dir = test_base_path / "a/b/c/c" + + test_dir.mkdir(parents=True) + test_py.write_text(code, encoding="UTF-8") + + pkg_root, module_name = resolve_pkg_root_and_module_name( + test_py, consider_namespace_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist2", + "com.company.a.b.c.test_demo", + ) + + result = pytester.runpytest("--import-mode=importlib", test_py) + + result.stdout.no_fnmatch_line( + "E KeyError: 'test_ns_multiple_levels_import1.src.dist2.com.company.a.b'" + ) + + assert "KeyError" not in result.stdout.str() + def test_is_importable(pytester: Pytester) -> None: pytester.syspathinsert() From a10949594357e15a0e411102f45575a42644d23b Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:27:10 +0800 Subject: [PATCH 2/9] chore: Remove unnecessary judgments and add more comments --- src/_pytest/pathlib.py | 61 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 0b3e861c707..1f620ee3894 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -616,12 +616,49 @@ def _import_module_using_spec( module_name: str, module_path: Path, module_location: Path, *, insert_modules: bool ) -> ModuleType | None: """ - Tries to import a module by its canonical name, path to the .py file, and its - parent location. + Tries to import a module by its canonical name, path, and its parent location. + + :param module_name: + The expected module name, will become the key of `sys.modules`. + + :param module_path: + The file path of the module, for example `test_demo.py`, + if module is a package, use the path of `__init__.py` in the package, + if module is a namespace package, use the path of directory. + + :param module_location: + The parent location of the module + if module is a package, usually the parent directory of the `__init__.py` path's parent directory. :param insert_modules: If True, will call insert_missing_modules to create empty intermediate modules for made-up module names (when importing test files not reachable from sys.path). + + Example 1 of parent_module_*: + module_name: a.b.c.demo + module_path: a/b/c/demo.py + module_location: a/b/c/ + if a.b.c is package (The file exists: a/b/c/__init__.py), then + parent_module_name: a.b.c + parent_module_path: a/b/c/__init__.py + parent_module_location: a/b/c/ + else: + parent_module_name: a.b.c + parent_module_path: a/b/c + parent_module_location: a/b/ + + Example 2 of parent_module_*: + module_name: a.b.c + module_path: a/b/c/__init__.py + module_location: a/b/c/ + if a.b is package (The file exists: a/b/__init__.py), then + parent_module_name: a.b + parent_module_path: a/b/__init__.py + parent_module_location: a/b/ + else: + parent_module_name: a.b + parent_module_path: a/b/ + parent_module_location: a/ """ # Attempt to import the parent module, seems is our responsibility: # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311 @@ -630,19 +667,21 @@ def _import_module_using_spec( if parent_module_name: parent_module = sys.modules.get(parent_module_name) if parent_module is None: - if (module_path.parent / "__init__.py").is_file(): - parent_module_location = module_location - else: - parent_module_location = module_location.parent - + # Get parent_location based on location, get parent_path based on path. if module_path.name == "__init__.py": + # If the current module is in a package, + # need to leave the package first and then enter the parent module. parent_module_path = module_path.parent.parent else: parent_module_path = module_path.parent - # Consider the parent module path as its __init__.py file, if it has one. - if (parent_module_path / "__init__.py").is_file(): + if (module_path.parent / "__init__.py").is_file(): + # If the parent module is a package, there is no need to change the location. + # because the __init__.py file can be found in the same directory. + parent_module_location = module_location parent_module_path = parent_module_path / "__init__.py" + else: + parent_module_location = module_location.parent parent_module = _import_module_using_spec( parent_module_name, @@ -661,9 +700,9 @@ def _import_module_using_spec( break else: loader = None - if "." in module_path.name and module_path.is_dir(): + if module_path.is_dir(): # The `spec_from_file_location` matches a loader based on the file extension by default. - # For a namespace package, you need to manually specify a loader. + # For a namespace package, need to manually specify a loader. loader = NamespaceLoader(name, module_path, PathFinder()) spec = importlib.util.spec_from_file_location( From 9d0fa23f8f5edadaccb0d2baf2241bb99c1dfde9 Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:12:17 +0800 Subject: [PATCH 3/9] add new test --- src/_pytest/pathlib.py | 10 +++------ testing/test_pathlib.py | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 1f620ee3894..83a4ae2852d 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -675,18 +675,14 @@ def _import_module_using_spec( else: parent_module_path = module_path.parent - if (module_path.parent / "__init__.py").is_file(): - # If the parent module is a package, there is no need to change the location. - # because the __init__.py file can be found in the same directory. - parent_module_location = module_location + if (parent_module_path / "__init__.py").is_file(): + # If the parent module is a package, loading by __init__.py file. parent_module_path = parent_module_path / "__init__.py" - else: - parent_module_location = module_location.parent parent_module = _import_module_using_spec( parent_module_name, parent_module_path, - parent_module_location, + parent_module_path.parent, insert_modules=insert_modules, ) diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index f907cd7310f..fc5ea1f9a74 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -18,6 +18,7 @@ import unittest.mock from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import _import_module_using_spec from _pytest.pathlib import bestrelpath from _pytest.pathlib import commonpath from _pytest.pathlib import compute_module_name @@ -36,6 +37,7 @@ from _pytest.pathlib import resolve_package_path from _pytest.pathlib import resolve_pkg_root_and_module_name from _pytest.pathlib import safe_exists +from _pytest.pathlib import spec_matches_module_path from _pytest.pathlib import symlink_or_skip from _pytest.pathlib import visit from _pytest.pytester import Pytester @@ -780,6 +782,51 @@ def test_insert_missing_modules( insert_missing_modules(modules, "") assert modules == {} + @pytest.mark.parametrize("b_is_package", [True, False]) + @pytest.mark.parametrize("insert_modules", [True, False]) + def test_import_module_using_spec( + self, b_is_package, insert_modules, tmp_path: Path + ): + """ + Verify that `_import_module_using_spec` can obtain a spec based on the path,thereby enabling the import. + When importing, not only the target module is imported, but also the parent modules are recursively imported. + """ + file_path = tmp_path / "a/b/c/demo.py" + file_path.parent.mkdir(parents=True) + file_path.touch() + + if b_is_package: + (tmp_path / "a/b/__init__.py").touch() + + mod = _import_module_using_spec( + "a.b.c.demo", + file_path, + file_path.parent, + insert_modules=insert_modules, + ) + + # target module is imported + assert mod is not None + assert spec_matches_module_path(mod.__spec__, file_path) is True + mod_demo = sys.modules["a.b.c.demo"] + assert "demo.py" in str(mod_demo) + + # parent modules are recursively imported. + mod_a = sys.modules["a"] + mod_b = sys.modules["a.b"] + mod_c = sys.modules["a.b.c"] + + assert mod_a.b is mod_b + assert mod_a.b.c is mod_c + assert mod_a.b.c.demo is mod_demo + + assert "namespace" in str(mod_a) + assert "namespace" in str(mod_c) + if b_is_package: + assert "namespace" not in str(mod_b) + else: + assert "namespace" in str(mod_b) + def test_parent_contains_child_module_attribute( self, monkeypatch: MonkeyPatch, tmp_path: Path ): From 48db28b7c55598a80345648115e12eb51a9128ad Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:38:05 +0800 Subject: [PATCH 4/9] update test compatible with Python<=3.11 --- testing/test_pathlib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index fc5ea1f9a74..19ec1fb5c7b 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -820,12 +820,12 @@ def test_import_module_using_spec( assert mod_a.b.c is mod_c assert mod_a.b.c.demo is mod_demo - assert "namespace" in str(mod_a) - assert "namespace" in str(mod_c) + assert "namespace" in str(mod_a).lower() + assert "namespace" in str(mod_c).lower() if b_is_package: - assert "namespace" not in str(mod_b) + assert "namespace" not in str(mod_b).lower() else: - assert "namespace" in str(mod_b) + assert "namespace" in str(mod_b).lower() def test_parent_contains_child_module_attribute( self, monkeypatch: MonkeyPatch, tmp_path: Path From 8dd554499b271c23077f4766c9ae53fd8a4d243a Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Mon, 9 Sep 2024 19:15:50 +0800 Subject: [PATCH 5/9] =?UTF-8?q?update=20test=EF=BC=9A=20verify=20the=20val?= =?UTF-8?q?idity=20of=20the=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testing/test_pathlib.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 19ec1fb5c7b..6a87da0cb4c 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -793,10 +793,10 @@ def test_import_module_using_spec( """ file_path = tmp_path / "a/b/c/demo.py" file_path.parent.mkdir(parents=True) - file_path.touch() + file_path.write_text("my_name='demo'") if b_is_package: - (tmp_path / "a/b/__init__.py").touch() + (tmp_path / "a/b/__init__.py").write_text("my_name='b.__init__'") mod = _import_module_using_spec( "a.b.c.demo", @@ -808,8 +808,10 @@ def test_import_module_using_spec( # target module is imported assert mod is not None assert spec_matches_module_path(mod.__spec__, file_path) is True + mod_demo = sys.modules["a.b.c.demo"] assert "demo.py" in str(mod_demo) + assert mod_demo.my_name == "demo" # Imported and available for use # parent modules are recursively imported. mod_a = sys.modules["a"] @@ -822,10 +824,17 @@ def test_import_module_using_spec( assert "namespace" in str(mod_a).lower() assert "namespace" in str(mod_c).lower() + + # Compatibility package and namespace package if b_is_package: assert "namespace" not in str(mod_b).lower() + assert "__init__.py" in str(mod_b).lower() # Imported __init__.py + assert mod_b.my_name == "b.__init__" # Imported and available for use + else: assert "namespace" in str(mod_b).lower() + with pytest.raises(AttributeError): # Not imported __init__.py + assert mod_b.my_name def test_parent_contains_child_module_attribute( self, monkeypatch: MonkeyPatch, tmp_path: Path From 60a7478c2782e42da4647dd5a5b8f65230a83a68 Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Mon, 9 Sep 2024 22:08:51 +0800 Subject: [PATCH 6/9] fix: ending parameter is required in ci/cd --- testing/test_pathlib.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 6a87da0cb4c..2f795bdfc75 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -793,10 +793,12 @@ def test_import_module_using_spec( """ file_path = tmp_path / "a/b/c/demo.py" file_path.parent.mkdir(parents=True) - file_path.write_text("my_name='demo'") + file_path.write_text("my_name='demo'", encoding="utf-8") if b_is_package: - (tmp_path / "a/b/__init__.py").write_text("my_name='b.__init__'") + (tmp_path / "a/b/__init__.py").write_text( + "my_name='b.__init__'", encoding="utf-8" + ) mod = _import_module_using_spec( "a.b.c.demo", From 6db16326936ed075fec24487b7bfcf25043725cc Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 21 Sep 2024 09:30:25 -0300 Subject: [PATCH 7/9] Minor tweaks --- src/_pytest/pathlib.py | 70 ++++++++++++++++++++++------------------- testing/test_pathlib.py | 11 +++---- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index 83a4ae2852d..dd36559ce1b 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -616,49 +616,51 @@ def _import_module_using_spec( module_name: str, module_path: Path, module_location: Path, *, insert_modules: bool ) -> ModuleType | None: """ - Tries to import a module by its canonical name, path, and its parent location. + Tries to import a module by its canonical name, path, and its parent location. :param module_name: The expected module name, will become the key of `sys.modules`. :param module_path: - The file path of the module, for example `test_demo.py`, - if module is a package, use the path of `__init__.py` in the package, - if module is a namespace package, use the path of directory. + The file path of the module, for example `/foo/bar/test_demo.py`. + If module is a package, pass the path to the `__init__.py` of the package. + If module is a namespace package, pass directory path. :param module_location: - The parent location of the module - if module is a package, usually the parent directory of the `__init__.py` path's parent directory. + The parent location of the module. + If module is a package, pass the directory containing the `__init__.py` file. :param insert_modules: - If True, will call insert_missing_modules to create empty intermediate modules - for made-up module names (when importing test files not reachable from sys.path). + If True, will call `insert_missing_modules` to create empty intermediate modules + with made-up module names (when importing test files not reachable from `sys.path`). Example 1 of parent_module_*: - module_name: a.b.c.demo - module_path: a/b/c/demo.py - module_location: a/b/c/ - if a.b.c is package (The file exists: a/b/c/__init__.py), then - parent_module_name: a.b.c - parent_module_path: a/b/c/__init__.py - parent_module_location: a/b/c/ + + module_name: "a.b.c.demo" + module_path: Path("a/b/c/demo.py") + module_location: Path("a/b/c/") + if "a.b.c" is package ("a/b/c/__init__.py" exists), then + parent_module_name: "a.b.c" + parent_module_path: Path("a/b/c/__init__.py") + parent_module_location: Path("a/b/c/") else: - parent_module_name: a.b.c - parent_module_path: a/b/c - parent_module_location: a/b/ + parent_module_name: "a.b.c" + parent_module_path: Path("a/b/c") + parent_module_location: Path("a/b/") Example 2 of parent_module_*: - module_name: a.b.c - module_path: a/b/c/__init__.py - module_location: a/b/c/ - if a.b is package (The file exists: a/b/__init__.py), then - parent_module_name: a.b - parent_module_path: a/b/__init__.py - parent_module_location: a/b/ + + module_name: "a.b.c" + module_path: Path("a/b/c/__init__.py") + module_location: Path("a/b/c/") + if "a.b" is package ("a/b/__init__.py" exists), then + parent_module_name: "a.b" + parent_module_path: Path("a/b/__init__.py") + parent_module_location: Path("a/b/") else: - parent_module_name: a.b - parent_module_path: a/b/ - parent_module_location: a/ + parent_module_name: "a.b" + parent_module_path: Path("a/b/") + parent_module_location: Path("a/") """ # Attempt to import the parent module, seems is our responsibility: # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311 @@ -719,6 +721,7 @@ def _import_module_using_spec( if insert_modules: insert_missing_modules(sys.modules, module_name) return mod + return None @@ -730,11 +733,12 @@ def spec_matches_module_path(module_spec: ModuleSpec | None, module_path: Path) if module_spec.origin: return Path(module_spec.origin) == module_path - # If this is a namespace package, compare the path with the `module_spec.submodule_Search_Locations` - # https://docs.python.org/zh-cn/3/library/importlib.html#importlib.machinery.ModuleSpec.submodule_search_locations - if module_spec.submodule_search_locations: - for _path in module_spec.submodule_search_locations: - if Path(_path) == module_path: + # Compare the path with the `module_spec.submodule_Search_Locations` in case + # the module is part of a namespace package. + # https://docs.python.org/3/library/importlib.html#importlib.machinery.ModuleSpec.submodule_search_locations + if module_spec.submodule_search_locations: # can be None. + for path in module_spec.submodule_search_locations: + if Path(path) == module_path: return True return False diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 2f795bdfc75..9fb4624339f 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -17,6 +17,7 @@ from typing import Sequence import unittest.mock +from _pytest.config import ExitCode from _pytest.monkeypatch import MonkeyPatch from _pytest.pathlib import _import_module_using_spec from _pytest.pathlib import bestrelpath @@ -788,7 +789,7 @@ def test_import_module_using_spec( self, b_is_package, insert_modules, tmp_path: Path ): """ - Verify that `_import_module_using_spec` can obtain a spec based on the path,thereby enabling the import. + Verify that `_import_module_using_spec` can obtain a spec based on the path, thereby enabling the import. When importing, not only the target module is imported, but also the parent modules are recursively imported. """ file_path = tmp_path / "a/b/c/demo.py" @@ -827,7 +828,7 @@ def test_import_module_using_spec( assert "namespace" in str(mod_a).lower() assert "namespace" in str(mod_c).lower() - # Compatibility package and namespace package + # Compatibility package and namespace package. if b_is_package: assert "namespace" not in str(mod_b).lower() assert "__init__.py" in str(mod_b).lower() # Imported __init__.py @@ -1616,9 +1617,7 @@ def test(): # along with a tests file test_base_path = tmp_path / "src/dist2/com/company" test_py = test_base_path / "a/b/c/test_demo.py" - test_dir = test_base_path / "a/b/c/c" - - test_dir.mkdir(parents=True) + test_py.parent.mkdir(parents=True) test_py.write_text(code, encoding="UTF-8") pkg_root, module_name = resolve_pkg_root_and_module_name( @@ -1630,7 +1629,7 @@ def test(): ) result = pytester.runpytest("--import-mode=importlib", test_py) - + assert result.ret == ExitCode.TESTS_FAILED result.stdout.no_fnmatch_line( "E KeyError: 'test_ns_multiple_levels_import1.src.dist2.com.company.a.b'" ) From 63c02c59e608bf3c3789d4d9063f1b28f22c6f61 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 21 Sep 2024 09:35:13 -0300 Subject: [PATCH 8/9] Tweak changelog --- changelog/12592.bugfix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/12592.bugfix.rst b/changelog/12592.bugfix.rst index 74a4cb22011..605783bcab4 100644 --- a/changelog/12592.bugfix.rst +++ b/changelog/12592.bugfix.rst @@ -1 +1 @@ -Fixed the issue of causes ``KeyError`` when using the parameter ``--import-mode=importlib`` in pytest>=8.2 . +Fixed :class:`KeyError` crash when using ``--import-mode=importlib`` in a directory layout where a directory contains a child directory with the same name. From 7b7105f9e1be72ae0d317d2a4c7892fffc5f896d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 21 Sep 2024 09:47:40 -0300 Subject: [PATCH 9/9] Simplify test to reflect the reported issue --- testing/test_pathlib.py | 45 ++++++++++------------------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 9fb4624339f..62359303f3b 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -1600,41 +1600,18 @@ def test_full_ns_packages_without_init_files( tmp_path / "src/dist2/ns/a/core/foo/m.py", consider_namespace_packages=True ) == (tmp_path / "src/dist2", "ns.a.core.foo.m") - def test_ns_multiple_levels_import_same_name_directory( - self, - tmp_path: Path, - monkeypatch: MonkeyPatch, - pytester: Pytester, - ) -> None: - """Check KeyError with `--import-mode=importlib` (#12592).""" - self.setup_directories(tmp_path, monkeypatch, pytester) - code = dedent(""" - def test(): - assert "four lights" == "five lights" - """) - - # a subdirectories with the same name in the namespace package, - # along with a tests file - test_base_path = tmp_path / "src/dist2/com/company" - test_py = test_base_path / "a/b/c/test_demo.py" - test_py.parent.mkdir(parents=True) - test_py.write_text(code, encoding="UTF-8") - - pkg_root, module_name = resolve_pkg_root_and_module_name( - test_py, consider_namespace_packages=True - ) - assert (pkg_root, module_name) == ( - tmp_path / "src/dist2", - "com.company.a.b.c.test_demo", - ) - - result = pytester.runpytest("--import-mode=importlib", test_py) - assert result.ret == ExitCode.TESTS_FAILED - result.stdout.no_fnmatch_line( - "E KeyError: 'test_ns_multiple_levels_import1.src.dist2.com.company.a.b'" - ) - assert "KeyError" not in result.stdout.str() +def test_ns_import_same_name_directory_12592( + tmp_path: Path, pytester: Pytester +) -> None: + """Regression for `--import-mode=importlib` with directory parent and child with same name (#12592).""" + y_dir = tmp_path / "x/y/y" + y_dir.mkdir(parents=True) + test_y = tmp_path / "x/y/test_y.py" + test_y.write_text("def test(): pass", encoding="UTF-8") + + result = pytester.runpytest("--import-mode=importlib", test_y) + assert result.ret == ExitCode.OK def test_is_importable(pytester: Pytester) -> None: