From 88503768b7bf6f0a4944dcf9339e0fefe1884c3b Mon Sep 17 00:00:00 2001 From: Richard Si Date: Sun, 20 Jul 2025 15:08:53 -0400 Subject: [PATCH 1/4] Support Direct URL editable requirements This is crucial for the deprecation of non-bare egg URL fragments as they used to be the sole way to request an extra for a VCS editable install. I've also removed all references to egg fragments from the user docs and added further explanation of the Direct URL form in the VCS support topic. Getting the error handling right was a pain. That's why I extracted the two parsing flows into their own functions. Co-authored-by: Tzu-ping Chung --- docs/html/cli/pip_install.rst | 24 ++++---- docs/html/topics/vcs-support.md | 38 ++++++++++--- docs/html/user_guide.rst | 2 +- news/13495.feature.rst | 1 + src/pip/_internal/req/constructors.py | 56 ++++++++++++++----- .../resolution/resolvelib/candidates.py | 6 +- tests/functional/test_install_reqs.py | 50 +++++++++++++++++ tests/unit/test_req.py | 19 +++++++ 8 files changed, 159 insertions(+), 37 deletions(-) create mode 100644 news/13495.feature.rst diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index 00d7f7d23b1..ff1dca9f4e9 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -66,8 +66,8 @@ for the name and project version (this is in theory slightly less reliable than using the ``egg_info`` command, but avoids downloading and processing unnecessary numbers of files). -Any URL may use the ``#egg=name`` syntax (see :doc:`../topics/vcs-support`) to -explicitly state the project name. +The :pep:`508` requirement syntax can be used to explicitly state the project +name (see :doc:`../topics/vcs-support`). Satisfying Requirements ----------------------- @@ -367,21 +367,21 @@ Examples .. code-block:: shell - python -m pip install -e 'git+https://git.repo/some_pkg.git#egg=SomePackage' # from git - python -m pip install -e 'hg+https://hg.repo/some_pkg.git#egg=SomePackage' # from mercurial - python -m pip install -e 'svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage' # from svn - python -m pip install -e 'git+https://git.repo/some_pkg.git@feature#egg=SomePackage' # from 'feature' branch - python -m pip install -e 'git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path' # install a python package from a repo subdirectory + python -m pip install -e 'SomePackage @ git+https://git.repo/some_pkg.git' # from git + python -m pip install -e 'SomePackage @ hg+https://hg.repo/some_pkg.git' # from mercurial + python -m pip install -e 'SomePakcage @ svn+svn://svn.repo/some_pkg/trunk/' # from svn + python -m pip install -e 'SomePackage @ git+https://git.repo/some_pkg.git@feature' # from 'feature' branch + python -m pip install -e 'subdir @ git+https://git.repo/some_repo.git#subdirectory=subdir_path' # install a python package from a repo subdirectory .. tab:: Windows .. code-block:: shell - py -m pip install -e "git+https://git.repo/some_pkg.git#egg=SomePackage" # from git - py -m pip install -e "hg+https://hg.repo/some_pkg.git#egg=SomePackage" # from mercurial - py -m pip install -e "svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage" # from svn - py -m pip install -e "git+https://git.repo/some_pkg.git@feature#egg=SomePackage" # from 'feature' branch - py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + py -m pip install -e "SomePackage @ git+https://git.repo/some_pkg.git" # from git + py -m pip install -e "SomePackage @ hg+https://hg.repo/some_pkg.git" # from mercurial + py -m pip install -e "SomePackage @ svn+svn://svn.repo/some_pkg/trunk/" # from svn + py -m pip install -e "SomePackage @ git+https://git.repo/some_pkg.git@feature" # from 'feature' branch + py -m pip install -e "subdir @ git+https://git.repo/some_repo.git#subdirectory=subdir_path" # install a python package from a repo subdirectory #. Install a package with extras, i.e., optional dependencies (:ref:`specification `). diff --git a/docs/html/topics/vcs-support.md b/docs/html/topics/vcs-support.md index c8169dbe24c..d637d66f00e 100644 --- a/docs/html/topics/vcs-support.md +++ b/docs/html/topics/vcs-support.md @@ -10,6 +10,25 @@ control system being used). It is used through URL prefixes: - Subversion -- `svn+` - Bazaar -- `bzr+` +The general form of a VCS requirement is `ProjectName @ VCS_URL`, e.g. + +```none +MyProject @ git+https://git.example.com/MyProject +MyProject[extra] @ git+https:/git.example.com/MyProject +``` + +This is the Direct URL ({pep}`508`) requirement syntax. It is also permissible +to remove `MyProject @` portion is removed and provide a bare VCS URL. + +```none +git+https://git.example.com/MyProject +``` + +This is a pip specific extension. This form can be used as long as pip does +not need to know the project name in advance. pip is generally able to infer +the project name except in the case of {ref}`editable-vcs-installs`. In +addition, extras cannot be requested using a bare VCS URL. + ## Supported VCS ### Git @@ -81,8 +100,8 @@ MyProject @ svn+ssh://user@svn.example.com/MyProject You can also give specific revisions to an SVN URL, like so: ```none --e svn+http://svn.example.com/svn/MyProject/trunk@2019#egg=MyProject --e svn+http://svn.example.com/svn/MyProject/trunk@{20080101}#egg=MyProject +-e MyProject @ svn+http://svn.example.com/svn/MyProject/trunk@2019 +-e MyProject @ svn+http://svn.example.com/svn/MyProject/trunk@{20080101} ``` Note that you need to use [Editable VCS installs](#editable-vcs-installs) for @@ -115,6 +134,9 @@ MyProject @ bzr+http://bzr.example.com/MyProject/trunk@v1.0 VCS projects can be installed in {ref}`editable mode ` (using the {ref}`--editable ` option) or not. +In editable mode, the project name must be provided upfront using the Direct URL +(`MyProject @ URL`) form so pip can determine the VCS clone location. + - The default clone location (for editable installs) is: - `/src/SomeProject` in virtual environments @@ -133,15 +155,15 @@ take on the VCS requirement (not the commit itself). ## URL fragments pip looks at the `subdirectory` fragments of VCS URLs for specifying the path to the -Python package, when it is not in the root of the VCS directory. eg: `pkg_dir`. +Python package, when it is not in the root of the VCS directory. -pip also looks at the `egg` fragment specifying the "project name". In practice the -`egg` fragment is only required to help pip determine the VCS clone location in editable -mode. In all other circumstances, the `egg` fragment is not necessary and its use is -discouraged. +```{note} +pip also supports an `egg` fragment to specify the "project name". This is a legacy +feature and its use is discouraged in favour of the Direct URL ({pep}`508`) form. The `egg` fragment **should** be a bare {ref}`project name `. Anything else is not guaranteed to work. +``` ````{admonition} Example If your repository layout is: @@ -164,6 +186,6 @@ $ pip install "pkg @ vcs+protocol://repo_url/#subdirectory=pkg_dir" or: ```{pip-cli} -$ pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" +$ pip install -e "pkg @ vcs+protocol://repo_url/#subdirectory=pkg_dir" ``` ```` diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index d6a0acf9cd8..f1d00279ca5 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -183,7 +183,7 @@ In practice, there are 4 common uses of Requirements files: ``sometag``. You'd reference it in your requirements file with a line like so:: - git+https://myvcs.com/some_dependency@sometag#egg=SomeDependency + SomeDependency @ git+https://myvcs.com/some_dependency@sometag If ``SomeDependency`` was previously a top-level requirement in your requirements file, then **replace** that line with the new line. If diff --git a/news/13495.feature.rst b/news/13495.feature.rst new file mode 100644 index 00000000000..911a086e192 --- /dev/null +++ b/news/13495.feature.rst @@ -0,0 +1 @@ +Add support installing an editable requirement written as a Direct URL. diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 056e7e3a7f1..a28c8cbc9da 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -86,17 +86,25 @@ def _set_requirement_extras(req: Requirement, new_extras: set[str]) -> Requireme return get_requirement(f"{pre}{extras}{post}") -def parse_editable(editable_req: str) -> tuple[str | None, str, set[str]]: - """Parses an editable requirement into: - - a requirement name - - an URL - - extras - - editable options - Accepted requirements: - svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir - .[some_extra] - """ +def _parse_direct_url_editable(editable_req: str) -> tuple[str | None, str, set[str]]: + try: + req = Requirement(editable_req) + except InvalidRequirement: + pass + else: + if req.url: + # Join the marker back into the name part. This will be parsed out + # later into a Requirement again. + if req.marker: + name = f"{req.name} ; {req.marker}" + else: + name = req.name + return (name, req.url, req.extras) + + raise ValueError + +def _parse_pip_syntax_editable(editable_req: str) -> tuple[str | None, str, set[str]]: url = editable_req # If a file path is specified with extras, strip off the extras. @@ -122,9 +130,27 @@ def parse_editable(editable_req: str) -> tuple[str | None, str, set[str]]: url = f"{version_control}+{url}" break + return Link(url).egg_fragment, url, set() + + +def parse_editable(editable_req: str) -> tuple[str | None, str, set[str]]: + """Parses an editable requirement into: + - a requirement name + - an URL + - extras + Accepted requirements: + - svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir + - local_path[some_extra] + - Foobar[extra] @ svn+http://blahblah@rev#subdirectory=subdir ; markers + """ + try: + package_name, url, extras = _parse_direct_url_editable(editable_req) + except ValueError: + package_name, url, extras = _parse_pip_syntax_editable(editable_req) + link = Link(url) - if not link.is_vcs: + if not link.is_vcs and not link.url.startswith("file:"): backends = ", ".join(vcs.all_schemes) raise InstallationError( f"{editable_req} is not a valid editable requirement. " @@ -132,13 +158,13 @@ def parse_editable(editable_req: str) -> tuple[str | None, str, set[str]]: f"(beginning with {backends})." ) - package_name = link.egg_fragment - if not package_name: + # The project name can be inferred from local file URIs easily. + if not package_name and not link.url.startswith("file:"): raise InstallationError( f"Could not detect requirement name for '{editable_req}', " - "please specify one with #egg=your_package_name" + "please specify one with your_package_name @ URL" ) - return package_name, url, set() + return package_name, url, extras def check_first_requirement_in_file(filename: str) -> None: diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index a8315349791..07c19671374 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -85,8 +85,12 @@ def make_install_req_from_editable( link: Link, template: InstallRequirement ) -> InstallRequirement: assert template.editable, "template not editable" + if template.name: + req_string = f"{template.name} @ {link.url}" + else: + req_string = link.url ireq = install_req_from_editable( - link.url, + req_string, user_supplied=template.user_supplied, comes_from=template.comes_from, use_pep517=template.use_pep517, diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 3c5b6db4a68..e841fcfbf87 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -10,6 +10,7 @@ PipTestEnvironment, ResolverVariant, TestData, + _create_test_package, _create_test_package_with_subdirectory, create_basic_sdist_for_package, create_basic_wheel_for_package, @@ -941,3 +942,52 @@ def test_nonpep517_setuptools_import_failure(script: PipTestEnvironment) -> None exc_message = "ImportError: this 'setuptools' was intentionally poisoned" assert nice_message in result.stderr assert exc_message in result.stderr + + +class TestEditableDirectURL: + def test_install_local_project( + self, script: PipTestEnvironment, data: TestData, common_wheels: Path + ) -> None: + uri = (data.src / "simplewheel-2.0").as_uri() + script.pip( + "install", "--no-index", "-e", f"simplewheel @ {uri}", "-f", common_wheels + ) + script.assert_installed(simplewheel="2.0") + + def test_install_local_project_with_extra( + self, script: PipTestEnvironment, data: TestData, common_wheels: Path + ) -> None: + uri = (data.src / "requires_simple_extra").as_uri() + script.pip( + "install", + "--no-index", + "-e", + f"requires-simple-extra[extra] @ {uri}", + "-f", + common_wheels, + "-f", + data.packages, + ) + script.assert_installed(requires_simple_extra="0.1") + script.assert_installed(simple="1.0") + + def test_install_local_git_repo( + self, script: PipTestEnvironment, common_wheels: Path + ) -> None: + repo_path = _create_test_package(script.scratch_path, "simple") + url = "git+" + repo_path.as_uri() + script.pip( + "install", "--no-index", "-e", f"simple @ {url}", "-f", common_wheels + ) + script.assert_installed(simple="0.1") + + @pytest.mark.network + def test_install_remote_git_repo_with_extra( + self, script: PipTestEnvironment, data: TestData, common_wheels: Path + ) -> None: + req = "pip-test-package[extra] @ git+https://github.com/pypa/pip-test-package" + script.pip( + "install", "--no-index", "-e", req, "-f", common_wheels, "-f", data.packages + ) + script.assert_installed(pip_test_package="0.1.1") + script.assert_installed(simple="3.0") diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 0547131134e..f812f2c7ed9 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -853,6 +853,25 @@ def test_install_req_extend_extras( assert extended.permit_editable_wheels == req.permit_editable_wheels +@pytest.mark.parametrize( + "req_str, expected", + [ + ( + 'foo[extra] @ svn+http://foo ; os_name == "nt"', + ('foo ; os_name == "nt"', "svn+http://foo", {"extra"}), + ), + ( + "foo @ svn+http://foo", + ("foo", "svn+http://foo", set()), + ), + ], +) +def test_parse_editable_pep508( + req_str: str, expected: tuple[str, str, set[str]] +) -> None: + assert parse_editable(req_str) == expected + + @mock.patch("pip._internal.req.req_install.os.path.abspath") @mock.patch("pip._internal.req.req_install.os.path.exists") @mock.patch("pip._internal.req.req_install.os.path.isdir") From f241d838722bcd17faa9fe356d56e67e41a54caf Mon Sep 17 00:00:00 2001 From: Richard Si Date: Mon, 20 Oct 2025 18:28:28 -0400 Subject: [PATCH 2/4] Apply minor tweaks from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stéphane Bidoul --- docs/html/cli/pip_install.rst | 4 ++-- news/13495.feature.rst | 2 +- src/pip/_internal/req/constructors.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index ff1dca9f4e9..5e2273a382d 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -371,7 +371,7 @@ Examples python -m pip install -e 'SomePackage @ hg+https://hg.repo/some_pkg.git' # from mercurial python -m pip install -e 'SomePakcage @ svn+svn://svn.repo/some_pkg/trunk/' # from svn python -m pip install -e 'SomePackage @ git+https://git.repo/some_pkg.git@feature' # from 'feature' branch - python -m pip install -e 'subdir @ git+https://git.repo/some_repo.git#subdirectory=subdir_path' # install a python package from a repo subdirectory + python -m pip install -e 'SomePackage @ git+https://git.repo/some_repo.git#subdirectory=subdir_path' # install a python package from a repo subdirectory .. tab:: Windows @@ -381,7 +381,7 @@ Examples py -m pip install -e "SomePackage @ hg+https://hg.repo/some_pkg.git" # from mercurial py -m pip install -e "SomePackage @ svn+svn://svn.repo/some_pkg/trunk/" # from svn py -m pip install -e "SomePackage @ git+https://git.repo/some_pkg.git@feature" # from 'feature' branch - py -m pip install -e "subdir @ git+https://git.repo/some_repo.git#subdirectory=subdir_path" # install a python package from a repo subdirectory + py -m pip install -e "SomePackage @ git+https://git.repo/some_repo.git#subdirectory=subdir_path" # install a python package from a repo subdirectory #. Install a package with extras, i.e., optional dependencies (:ref:`specification `). diff --git a/news/13495.feature.rst b/news/13495.feature.rst index 911a086e192..c3536da0312 100644 --- a/news/13495.feature.rst +++ b/news/13495.feature.rst @@ -1 +1 @@ -Add support installing an editable requirement written as a Direct URL. +Add support installing an editable requirement written as a Direct URL (``PackageName @ URL``). diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index a28c8cbc9da..d7cddb0067c 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -135,7 +135,7 @@ def _parse_pip_syntax_editable(editable_req: str) -> tuple[str | None, str, set[ def parse_editable(editable_req: str) -> tuple[str | None, str, set[str]]: """Parses an editable requirement into: - - a requirement name + - a requirement name with environment markers - an URL - extras Accepted requirements: From 71e5a1263366e9508d0a6d9449ec133a902afce6 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Mon, 20 Oct 2025 18:46:30 -0400 Subject: [PATCH 3/4] Link to PyPUG --- docs/html/cli/pip_install.rst | 4 ++-- docs/html/topics/vcs-support.md | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index 5e2273a382d..474713f0674 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -66,8 +66,8 @@ for the name and project version (this is in theory slightly less reliable than using the ``egg_info`` command, but avoids downloading and processing unnecessary numbers of files). -The :pep:`508` requirement syntax can be used to explicitly state the project -name (see :doc:`../topics/vcs-support`). +The :ref:`Direct URL requirement syntax ` can be used +to explicitly state the project name (see :doc:`../topics/vcs-support`). Satisfying Requirements ----------------------- diff --git a/docs/html/topics/vcs-support.md b/docs/html/topics/vcs-support.md index d637d66f00e..be753f8c407 100644 --- a/docs/html/topics/vcs-support.md +++ b/docs/html/topics/vcs-support.md @@ -17,8 +17,9 @@ MyProject @ git+https://git.example.com/MyProject MyProject[extra] @ git+https:/git.example.com/MyProject ``` -This is the Direct URL ({pep}`508`) requirement syntax. It is also permissible -to remove `MyProject @` portion is removed and provide a bare VCS URL. +This is the {ref}`Direct URL ` requirement syntax. +It is also permissible to remove `MyProject @` portion is removed and provide +a bare VCS URL. ```none git+https://git.example.com/MyProject @@ -159,7 +160,8 @@ Python package, when it is not in the root of the VCS directory. ```{note} pip also supports an `egg` fragment to specify the "project name". This is a legacy -feature and its use is discouraged in favour of the Direct URL ({pep}`508`) form. +feature and its use is discouraged in favour of the +{ref}`Direct URL ` form. The `egg` fragment **should** be a bare {ref}`project name `. Anything else is not guaranteed to work. From 8a4d0493858f7d6dbd7b5149f77cf4df15e8b5bf Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Thu, 23 Oct 2025 19:51:26 -0400 Subject: [PATCH 4/4] Remove test_nonpep517_setuptools_import_failure test The --no-use-pep517 option has been removed from pip as part of the move to PEP 517-only builds. This test is no longer valid and was causing failures. Also removed the no-longer-needed make_wheel import. --- tests/functional/test_install_reqs.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 5d90e12c3c8..3492119fe8e 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -18,7 +18,6 @@ requirements_file, ) from tests.lib.local_repos import local_checkout -from tests.lib.wheel import make_wheel class ArgRecordingSdist: @@ -944,29 +943,6 @@ def test_config_settings_local_to_package( assert "--verbose" not in simple2_args -def test_nonpep517_setuptools_import_failure(script: PipTestEnvironment) -> None: - """Any import failures of `setuptools` should inform the user both that it's - not pip's fault, but also exactly what went wrong in the import.""" - # Install a poisoned version of 'setuptools' that fails to import. - name = "setuptools_poisoned" - module = """\ -raise ImportError("this 'setuptools' was intentionally poisoned") -""" - path = make_wheel(name, "0.1.0", extra_files={"setuptools.py": module}).save_to_dir( - script.scratch_path - ) - script.pip("install", "--no-index", path) - - result = script.pip_install_local("--no-use-pep517", "simple", expect_error=True) - nice_message = ( - "ERROR: Can not execute `setup.py`" - " since setuptools failed to import in the build environment" - ) - exc_message = "ImportError: this 'setuptools' was intentionally poisoned" - assert nice_message in result.stderr - assert exc_message in result.stderr - - class TestEditableDirectURL: def test_install_local_project( self, script: PipTestEnvironment, data: TestData, common_wheels: Path